CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2025-11-29 08:33:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9061 from freqtrade/fix/7389_backtest_startup_candle

Browse Source 
improve `get_analyzed_dataframe` behavior in early candles
This commit is contained in:
Matthias
2023-08-17 08:25:30 +02:00
committed by GitHub
parent b93c6235c1 452e1ab016
commit 7bc317fea7
4 changed files with 42 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
freqtrade/data/dataprovider.py
View File

@@ -17,7 +17,7 @@ from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWith
from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType, RPCMessageType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.exchange import Exchange, timeframe_to_prev_date, timeframe_to_seconds
from freqtrade.exchange.types import OrderBook
from freqtrade.misc import append_candles_to_dataframe
from freqtrade.rpc import RPCManager
@@ -46,6 +46,8 @@ class DataProvider:
self.__rpc = rpc
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None
self.__slice_date: Optional[datetime] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
self.__producer_pairs_df: Dict[str,
Dict[PairWithTimeframe, Tuple[DataFrame, datetime]]] = {}
@@ -64,10 +66,19 @@ class DataProvider:
def _set_dataframe_max_index(self, limit_index: int):
"""
Limit analyzed dataframe to max specified index.
Only relevant in backtesting.
:param limit_index: dataframe index.
"""
self.__slice_index = limit_index
def _set_dataframe_max_date(self, limit_date: datetime):
"""
Limit infomrative dataframe to max specified index.
Only relevant in backtesting.
:param limit_date: "current date"
"""
self.__slice_date = limit_date
def _set_cached_df(
self,
pair: str,
@@ -356,6 +367,11 @@ class DataProvider:
# Get historical OHLCV data (cached on disk).
timeframe = timeframe or self._config['timeframe']
data = self.historic_ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
# Cut date to timeframe-specific date.
# This is necessary to prevent lookahead bias in callbacks through informative pairs.
if self.__slice_date:
cutoff_date = timeframe_to_prev_date(timeframe, self.__slice_date)
data = data.loc[data['date'] < cutoff_date]
if len(data) == 0:
logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).")
return data

12
freqtrade/optimize/backtesting.py
View File

@@ -369,13 +369,14 @@ class Backtesting:
# Cleanup from prior runs
pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore')
df_analyzed = self.strategy.ft_advise_signals(pair_data, {'pair': pair})
# Trim startup period from analyzed dataframe
df_analyzed = processed[pair] = pair_data = trim_dataframe(
df_analyzed, self.timerange, startup_candles=self.required_startup)
# Update dataprovider cache
self.dataprovider._set_cached_df(
pair, self.timeframe, df_analyzed, self.config['candle_type_def'])
# Trim startup period from analyzed dataframe
df_analyzed = processed[pair] = pair_data = trim_dataframe(
df_analyzed, self.timerange, startup_candles=self.required_startup)
# Create a copy of the dataframe before shifting, that way the entry signal/tag
# remains on the correct candle for callbacks.
df_analyzed = df_analyzed.copy()
@@ -1196,7 +1197,8 @@ class Backtesting:
row_index += 1
indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index)
self.dataprovider._set_dataframe_max_index(self.required_startup + row_index)
self.dataprovider._set_dataframe_max_date(current_time)
current_detail_time: datetime = row[DATE_IDX].to_pydatetime()
trade_dir: Optional[LongShort] = self.check_for_trade_entry(row)
@@ -1229,12 +1231,14 @@ class Backtesting:
is_first = True
current_time_det = current_time
for det_row in detail_data[HEADERS].values.tolist():
self.dataprovider._set_dataframe_max_date(current_time_det)
open_trade_count_start = self.backtest_loop(
det_row, pair, current_time_det, end_date,
open_trade_count_start, trade_dir, is_first)
current_time_det += timedelta(minutes=self.timeframe_detail_min)
is_first = False
else:
self.dataprovider._set_dataframe_max_date(current_time)
open_trade_count_start = self.backtest_loop(
row, pair, current_time, end_date,
open_trade_count_start, trade_dir)

11
tests/data/test_dataprovider.py
View File

@@ -129,9 +129,14 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type):
default_conf["runmode"] = RunMode.BACKTEST
dp = DataProvider(default_conf, exchange)
assert dp.runmode == RunMode.BACKTEST
assert isinstance(dp.get_pair_dataframe(
"UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame)
# assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty
df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
assert isinstance(df, DataFrame)
assert len(df) == 3 # ohlcv_history mock has just 3 rows
dp._set_dataframe_max_date(ohlcv_history.iloc[-1]['date'])
df = dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)
assert isinstance(df, DataFrame)
assert len(df) == 2 # ohlcv_history is limited to 2 rows now
def test_available_pairs(mocker, default_conf, ohlcv_history):

12
tests/optimize/test_backtesting.py
View File

@@ -20,7 +20,7 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import get_timerange
from freqtrade.enums import CandleType, ExitType, RunMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.exchange import timeframe_to_next_date, timeframe_to_prev_date
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, get_strategy_run_id
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.persistence import LocalTrade, Trade
@@ -1135,6 +1135,12 @@ def test_backtest_dataprovider_analyzed_df(default_conf, fee, mocker, testdatadi
assert candle_date == current_time
# These asserts don't properly raise as they are nested,
# therefore we increment count and assert for that.
df = dp.get_pair_dataframe(pair, backtesting.strategy.timeframe)
prior_time = timeframe_to_prev_date(backtesting.strategy.timeframe,
candle_date - timedelta(seconds=1))
assert prior_time == df.iloc[-1].squeeze()['date']
assert df.iloc[-1].squeeze()['date'] < current_time
count += 1
backtesting.strategy.confirm_trade_entry = tmp_confirm_entry
@@ -1353,11 +1359,11 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
# Cached data correctly removed amounts
offset = 1 if tres == 0 else 0
removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count
removed_candles = len(data[pair]) - offset
assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles
assert len(
backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m')[0]
) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count
) == len(data['NXT/BTC']) - 1
backtesting.strategy.max_open_trades = 1
backtesting.config.update({'max_open_trades': 1})