diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 3a6f74b97..10569e7c7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -9,7 +9,7 @@ from collections import deque from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple -from pandas import DataFrame, concat, to_timedelta +from pandas import DataFrame, to_timedelta from freqtrade.configuration import TimeRange from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe @@ -191,30 +191,13 @@ class DataProvider: existing_df, _ = self.__producer_pairs_df[producer_name][pair_key] - # Handle overlapping candles - old_candles = existing_df[ - ~existing_df['date'].isin( - dataframe['date'] - ) - ] - overlapping_candles = existing_df[ - existing_df['date'].isin( - dataframe['date'] - ) - ] - new_candles = dataframe[ - ~dataframe['date'].isin( - existing_df['date'] - ) - ] - - if overlapping_candles: - existing_df = concat([old_candles, overlapping_candles], axis=0) - # CHECK FOR MISSING CANDLES timeframe_delta = to_timedelta(timeframe) # Convert the timeframe to a timedelta for pandas local_last = existing_df.iloc[-1]['date'] # We want the last date from our copy - incoming_first = new_candles.iloc[0]['date'] # We want the first date from the incoming + incoming_first = dataframe.iloc[0]['date'] # We want the first date from the incoming + + # Remove existing candles that are newer than the incoming first candle + existing_df1 = existing_df[existing_df['date'] < incoming_first] candle_difference = (incoming_first - local_last) / timeframe_delta @@ -225,8 +208,10 @@ class DataProvider: # so return False and candle_difference. if candle_difference > 1: return (False, candle_difference) - - appended_df = append_candles_to_dataframe(existing_df, dataframe) + if existing_df1.empty: + appended_df = dataframe + else: + appended_df = append_candles_to_dataframe(existing_df1, dataframe) # Everything is good, we appended self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed