Removes clean_duplicate_trades

freqtrade/data/converter/__init__.py

@@ -1,6 +1,6 @@
-from freqtrade.data.converter.converter import (clean_duplicate_trades, clean_ohlcv_dataframe,
+from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format,
-                                                convert_ohlcv_format, ohlcv_fill_up_missing_data,
+                                                ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
-                                                ohlcv_to_dataframe, order_book_to_dataframe,
+                                                order_book_to_dataframe,
                                                 populate_dataframe_with_trades,
                                                 public_trades_to_dataframe,
                                                 reduce_dataframe_footprint, trim_dataframe,
@@ -13,7 +13,6 @@ from freqtrade.data.converter.trade_converter import (convert_trades_format,
 
 
 __all__ = [
-    'clean_duplicate_trades',
     'clean_ohlcv_dataframe',
     'convert_ohlcv_format',
     'ohlcv_fill_up_missing_data',

freqtrade/data/converter/converter.py

@@ -11,7 +11,6 @@ from pandas import DataFrame, to_datetime
 
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_ORDERFLOW_COLUMNS,
                                  DEFAULT_TRADES_COLUMNS, Config)
-from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates
 from freqtrade.enums import CandleType, TradingMode
 
 
@@ -353,43 +352,6 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
         return data
 
 
-def warn_of_tick_duplicates(data: DataFrame, pair: str) -> None:
-    no_dupes_colunms = ['id', 'timestamp', 'datetime']
-    for col in no_dupes_colunms:
-        if col in data.columns and data[col].duplicated().any():
-            sum = data[col].duplicated().sum()
-            message = f'{sum} duplicated ticks for {pair} in {col} detected.'
-            if col == 'id':
-                logger.warning(message)
-            else:
-                logger.debug(message)
-
-
-def clean_duplicate_trades(trades: DataFrame, timeframe: str, pair: str, *,
-
-                           fill_missing: bool, drop_incomplete: bool) -> DataFrame:
-    """
-    Cleanse a TRADES dataframe by
-      * Grouping it by date (removes duplicate tics)
-      * dropping last candles if requested
-      * Filling up missing data (if requested)
-    :param data: DataFrame containing candle (TRADES) data.
-    :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data
-    :param pair: Pair this data is for (used to warn if fillup was necessary)
-    :param fill_missing: fill up missing candles with 0 candles
-                         (see trades_fill_up_missing_data for details)
-    :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete
-    :return: DataFrame
-    """
-    # group by index and aggregate results to eliminate duplicate ticks
-    # check if data has duplicate ticks
-    logger.debug(f"Clean duplicated ticks from Trades data {pair}")
-    df = pd.DataFrame(trades_df_remove_duplicates(
-        trades), columns=trades.columns)
-
-    return df
-
-
 def drop_incomplete_and_fill_missing_trades(data: DataFrame, timeframe: str, pair: str, *,
                                             fill_missing: bool, drop_incomplete: bool) -> DataFrame:
 

freqtrade/exchange/exchange.py

@@ -14,6 +14,7 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
 
 import ccxt
 import ccxt.async_support as ccxt_async
+import pandas as pd
 from cachetools import TTLCache
 from ccxt import TICK_SIZE
 from dateutil import parser
@@ -25,7 +26,8 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, DEFAULT_TRADES_
                                  PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.data.converter.converter import (_calculate_ohlcv_candle_start_and_end,
-                                                clean_duplicate_trades, public_trades_to_dataframe)
+                                                public_trades_to_dataframe)
+from freqtrade.data.converter.trade_converter import trades_df_remove_duplicates
 from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
@@ -2115,13 +2117,10 @@ class Exchange:
             if (pair, timeframe, c_type) in self._trades:
                 old = self._trades[(pair, timeframe, c_type)]
                 # Reassign so we return the updated, combined df
-                trades_df = clean_duplicate_trades(concat(
+                combined_df = concat([old, trades_df], axis=0)
-                    [old, trades_df], axis=0),
+                logger.debug(f"Clean duplicated ticks from Trades data {pair}")
-                    timeframe,
+                trades_df = pd.DataFrame(trades_df_remove_duplicates(combined_df),
-                    pair,
+                                         columns=combined_df.columns)
-                    fill_missing=False,
-                    drop_incomplete=False)
-                # warn_of_tick_duplicates(trades_df, pair)
                 # Age out old candles
                 if first_required_candle_date:
                     # slice of older dates