diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 85a77fe5e..6dbb38bd8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -6,6 +6,7 @@ import asyncio import inspect import logging import signal +from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone from math import floor @@ -43,6 +44,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_ from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.util import dt_from_ts, dt_now from freqtrade.util.datetime_helpers import dt_humanize, dt_ts +from freqtrade.util.periodic_cache import PeriodicCache logger = logging.getLogger(__name__) @@ -131,6 +133,7 @@ class Exchange: # Holds candles self._klines: Dict[PairWithTimeframe, DataFrame] = {} + self._expiring_candle_cache: Dict[str, PeriodicCache] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -2124,6 +2127,39 @@ class Exchange: return results_df + def refresh_ohlcv_with_cache( + self, + pairs: List[PairWithTimeframe], + since_ms: int + ) -> Dict[PairWithTimeframe, DataFrame]: + """ + Refresh ohlcv data for all pairs in needed_pairs if necessary. + Caches data with expiring per timeframe. + Should only be used for pairlists which need "on time" expirarion, and no longer cache. + """ + + timeframes = [p[1] for p in pairs] + for timeframe in timeframes: + if timeframe not in self._expiring_candle_cache: + timeframe_in_sec = timeframe_to_seconds(timeframe) + # Initialise cache + self._expiring_candle_cache[timeframe] = PeriodicCache(ttl=timeframe_in_sec, + maxsize=1000) + + # Get candles from cache + candles = { + c: self._expiring_candle_cache[c[1]].get(c, None) for c in pairs + if c in self._expiring_candle_cache[c[1]] + } + pairs_to_download = [p for p in pairs if p not in candles] + if pairs_to_download: + candles = self.refresh_latest_ohlcv( + pairs_to_download, since_ms=since_ms, cache=False + ) + for c, val in candles.items(): + self._expiring_candle_cache[c[1]][c] = val + return candles + def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 671ba4db4..f4d08e800 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter -from freqtrade.util import PeriodicCache, dt_now, format_ms_time +from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) @@ -63,7 +63,6 @@ class VolumePairList(IPairList): # get timeframe in minutes and seconds self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) _tf_in_sec = self._tf_in_min * 60 - self._candle_cache = PeriodicCache(maxsize=1000, ttl=_tf_in_sec) # wether to use range lookback or not self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) @@ -230,18 +229,7 @@ class VolumePairList(IPairList): if p not in self._pair_cache ] - # Get all candles - candles = { - c: self._candle_cache.get(c, None) for c in needed_pairs - if c in self._candle_cache - } - pairs_to_download = [p for p in needed_pairs if p not in candles] - if pairs_to_download: - candles = self._exchange.refresh_latest_ohlcv( - pairs_to_download, since_ms=since_ms, cache=False - ) - for c, val in candles.items(): - self._candle_cache[c] = val + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) for i, p in enumerate(filtered_tickers): contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0