diff --git a/freqtrade/data/converter/__init__.py b/freqtrade/data/converter/__init__.py index 270b657be..204e63ea7 100644 --- a/freqtrade/data/converter/__init__.py +++ b/freqtrade/data/converter/__init__.py @@ -1,6 +1,6 @@ -from freqtrade.data.converter.converter import (clean_duplicate_trades, clean_ohlcv_dataframe, - convert_ohlcv_format, ohlcv_fill_up_missing_data, - ohlcv_to_dataframe, order_book_to_dataframe, +from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format, + ohlcv_fill_up_missing_data, ohlcv_to_dataframe, + order_book_to_dataframe, populate_dataframe_with_trades, public_trades_to_dataframe, reduce_dataframe_footprint, trim_dataframe, @@ -13,7 +13,6 @@ from freqtrade.data.converter.trade_converter import (convert_trades_format, __all__ = [ - 'clean_duplicate_trades', 'clean_ohlcv_dataframe', 'convert_ohlcv_format', 'ohlcv_fill_up_missing_data', diff --git a/freqtrade/data/converter/converter.py b/freqtrade/data/converter/converter.py index 2af41a603..7f0ad8660 100644 --- a/freqtrade/data/converter/converter.py +++ b/freqtrade/data/converter/converter.py @@ -11,7 +11,6 @@ from pandas import DataFrame, to_datetime from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_ORDERFLOW_COLUMNS, DEFAULT_TRADES_COLUMNS, Config) -from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates from freqtrade.enums import CandleType, TradingMode @@ -353,43 +352,6 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, return data -def warn_of_tick_duplicates(data: DataFrame, pair: str) -> None: - no_dupes_colunms = ['id', 'timestamp', 'datetime'] - for col in no_dupes_colunms: - if col in data.columns and data[col].duplicated().any(): - sum = data[col].duplicated().sum() - message = f'{sum} duplicated ticks for {pair} in {col} detected.' - if col == 'id': - logger.warning(message) - else: - logger.debug(message) - - -def clean_duplicate_trades(trades: DataFrame, timeframe: str, pair: str, *, - - fill_missing: bool, drop_incomplete: bool) -> DataFrame: - """ - Cleanse a TRADES dataframe by - * Grouping it by date (removes duplicate tics) - * dropping last candles if requested - * Filling up missing data (if requested) - :param data: DataFrame containing candle (TRADES) data. - :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data - :param pair: Pair this data is for (used to warn if fillup was necessary) - :param fill_missing: fill up missing candles with 0 candles - (see trades_fill_up_missing_data for details) - :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete - :return: DataFrame - """ - # group by index and aggregate results to eliminate duplicate ticks - # check if data has duplicate ticks - logger.debug(f"Clean duplicated ticks from Trades data {pair}") - df = pd.DataFrame(trades_df_remove_duplicates( - trades), columns=trades.columns) - - return df - - def drop_incomplete_and_fill_missing_trades(data: DataFrame, timeframe: str, pair: str, *, fill_missing: bool, drop_incomplete: bool) -> DataFrame: diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 95ce43887..cd4de447e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -14,6 +14,7 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union import ccxt import ccxt.async_support as ccxt_async +import pandas as pd from cachetools import TTLCache from ccxt import TICK_SIZE from dateutil import parser @@ -25,7 +26,8 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, DEFAULT_TRADES_ PairWithTimeframe) from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list from freqtrade.data.converter.converter import (_calculate_ohlcv_candle_start_and_end, - clean_duplicate_trades, public_trades_to_dataframe) + public_trades_to_dataframe) +from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, @@ -2115,13 +2117,10 @@ class Exchange: if (pair, timeframe, c_type) in self._trades: old = self._trades[(pair, timeframe, c_type)] # Reassign so we return the updated, combined df - trades_df = clean_duplicate_trades(concat( - [old, trades_df], axis=0), - timeframe, - pair, - fill_missing=False, - drop_incomplete=False) - # warn_of_tick_duplicates(trades_df, pair) + combined_df = concat([old, trades_df], axis=0) + logger.debug(f"Clean duplicated ticks from Trades data {pair}") + trades_df = pd.DataFrame(trades_df_remove_duplicates(combined_df), + columns=combined_df.columns) # Age out old candles if first_required_candle_date: # slice of older dates