From f705293353552e41c58f8a892c64db1c1f4f2bee Mon Sep 17 00:00:00 2001 From: George Muravei-Alkhavoi Date: Mon, 19 Jul 2021 00:25:24 +0300 Subject: [PATCH 1/3] Dataprovider caching and trimming to timerange of historical informative. --- freqtrade/data/dataprovider.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 391ed4587..aa3efbd60 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -14,7 +14,8 @@ from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException -from freqtrade.exchange import Exchange +from freqtrade.exchange import Exchange, timeframe_to_seconds +from freqtrade.configuration import TimeRange logger = logging.getLogger(__name__) @@ -31,6 +32,7 @@ class DataProvider: self._pairlists = pairlists self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__slice_index: Optional[int] = None + self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {} def _set_dataframe_max_index(self, limit_index: int): """ @@ -62,11 +64,20 @@ class DataProvider: :param pair: pair to get the data for :param timeframe: timeframe to get data for """ - return load_pair_history(pair=pair, - timeframe=timeframe or self._config['timeframe'], - datadir=self._config['datadir'], - data_format=self._config.get('dataformat_ohlcv', 'json') - ) + saved_pair = (pair, timeframe) + if saved_pair not in self.__cached_pairs_backtesting: + timerange = TimeRange.parse_timerange(None if self._config.get( + 'timerange') is None else str(self._config.get('timerange'))) + # Move informative start time respecting startup_candle_count + timerange.subtract_start(timeframe_to_seconds(timeframe) * self._config.get('startup_candle_count', 0)) + self.__cached_pairs_backtesting[saved_pair] = load_pair_history( + pair=pair, + timeframe=timeframe or self._config['timeframe'], + datadir=self._config['datadir'], + timerange=timerange, + data_format=self._config.get('dataformat_ohlcv', 'json') + ) + return self.__cached_pairs_backtesting[saved_pair].copy() def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: """ From ab786abf7f9ebfbea04b0e1e39cca105023a95ea Mon Sep 17 00:00:00 2001 From: George Muravei-Alkhavoi Date: Mon, 19 Jul 2021 00:47:51 +0300 Subject: [PATCH 2/3] Fix intendation. --- freqtrade/data/dataprovider.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index aa3efbd60..cdee0f078 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -10,12 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame +from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange, timeframe_to_seconds -from freqtrade.configuration import TimeRange logger = logging.getLogger(__name__) @@ -64,12 +64,14 @@ class DataProvider: :param pair: pair to get the data for :param timeframe: timeframe to get data for """ - saved_pair = (pair, timeframe) + saved_pair = (pair, str(timeframe)) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) # Move informative start time respecting startup_candle_count - timerange.subtract_start(timeframe_to_seconds(timeframe) * self._config.get('startup_candle_count', 0)) + timerange.subtract_start( + timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0) + ) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( pair=pair, timeframe=timeframe or self._config['timeframe'], From 1ea29a918a9fb55f32f39fd098389b6427e64e3d Mon Sep 17 00:00:00 2001 From: George Muravei-Alkhavoi Date: Wed, 21 Jul 2021 00:09:09 +0300 Subject: [PATCH 3/3] Fix webserver timerange problem. --- freqtrade/rpc/api_server/api_backtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 76b4a8169..2fa66645b 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -47,15 +47,15 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac not ApiServer._bt or lastconfig.get('timeframe') != strat.timeframe or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) + or lastconfig.get('timerange') != btconfig['timerange'] ): from freqtrade.optimize.backtesting import Backtesting ApiServer._bt = Backtesting(btconfig) - # Only reload data if timeframe or timerange changed. + # Only reload data if timeframe changed. if ( not ApiServer._bt_data or not ApiServer._bt_timerange - or lastconfig.get('timerange') != btconfig['timerange'] or lastconfig.get('stake_amount') != btconfig.get('stake_amount') or lastconfig.get('enable_protections') != btconfig.get('enable_protections') or lastconfig.get('protections') != btconfig.get('protections', [])