Merge pull request #12214 from mrpabloyeah/fix-shufflefilter-behavior-in-backtesting

Fix ShuffleFilter behavior in backtesting

freqtrade/commands/arguments.py

@@ -49,6 +49,7 @@ ARGS_BACKTEST = [
     *ARGS_COMMON_OPTIMIZE,
     "position_stacking",
     "enable_protections",
+    "enable_dynamic_pairlist",
     "dry_run_wallet",
     "timeframe_detail",
     "strategy_list",

freqtrade/commands/cli_options.py

@@ -184,12 +184,20 @@ AVAILABLE_CLI_OPTIONS = {
     "enable_protections": Arg(
         "--enable-protections",
         "--enableprotections",
-        help="Enable protections for backtesting."
+        help="Enable protections for backtesting. "
         "Will slow backtesting down by a considerable amount, but will include "
         "configured protections",
         action="store_true",
         default=False,
     ),
+    "enable_dynamic_pairlist": Arg(
+        "--enable-dynamic-pairlist",
+        help="Enables dynamic pairlist refreshes in backtesting. "
+        "The pairlist will be generated for each new candle if you're using a "
+        "pairlist handler that supports this feature, for example, ShuffleFilter.",
+        action="store_true",
+        default=False,
+    ),
     "strategy_list": Arg(
         "--strategy-list",
         help="Provide a space-separated list of strategies to backtest. "

freqtrade/configuration/configuration.py

@@ -259,7 +259,13 @@ class Configuration:
         self._args_to_config(
             config,
             argname="enable_protections",
-            logstring="Parameter --enable-protections detected, enabling Protections. ...",
+            logstring="Parameter --enable-protections detected, enabling Protections ...",
+        )
+
+        self._args_to_config(
+            config,
+            argname="enable_dynamic_pairlist",
+            logstring="Parameter --enable-dynamic-pairlist detected, enabling dynamic pairlist ...",
         )
 
         if self.args.get("max_open_trades"):

freqtrade/optimize/backtesting.py

@@ -211,6 +211,7 @@ class Backtesting:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get("position_stacking", False)
         self.enable_protections: bool = self.config.get("enable_protections", False)
+        self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
         migrate_data(config, self.exchange)
 
         self.init_backtest()
@@ -1584,6 +1585,11 @@ class Backtesting:
         for current_time in self._time_generator(start_date, end_date):
             # Loop for each main candle.
             self.check_abort()
+
+            if self.dynamic_pairlist and self.pairlists:
+                self.pairlists.refresh_pairlist()
+                pairs = self.pairlists.whitelist
+
             # Reset open trade count for this candle
             # Critical to avoid exceeding max_open_trades in backtesting
             # when timeframe-detail is used and trades close within the opening candle.

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -93,6 +93,8 @@ class ShuffleFilter(IPairList):
             return pairlist_new
         # Shuffle is done inplace
         self._random.shuffle(pairlist)
-        self.__pairlist_cache[pairlist_bef] = pairlist
+
+        if self._config.get("runmode") in (RunMode.LIVE, RunMode.DRY_RUN):
+            self.__pairlist_cache[pairlist_bef] = pairlist
 
         return pairlist

freqtrade/plugins/pairlist/StaticPairList.py

@@ -7,6 +7,9 @@ Provides pair white list as it configured in config
 import logging
 from copy import deepcopy
 
+from cachetools import LRUCache
+
+from freqtrade.enums import RunMode
 from freqtrade.exchange.exchange_types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
@@ -22,6 +25,8 @@ class StaticPairList(IPairList):
         super().__init__(*args, **kwargs)
 
         self._allow_inactive = self._pairlistconfig.get("allow_inactive", False)
+        # Pair cache - only used for optimize modes
+        self._bt_pair_cache: LRUCache = LRUCache(maxsize=1)
 
     @property
     def needstickers(self) -> bool:
@@ -60,15 +65,23 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: List of pairs
         """
-        wl = self.verify_whitelist(
-            self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
-        )
-        if self._allow_inactive:
-            return wl
-        else:
-            # Avoid implicit filtering of "verify_whitelist" to keep
-            # proper warnings in the log
-            return self._whitelist_for_active_markets(wl)
+        pairlist = self._bt_pair_cache.get("pairlist")
+
+        if not pairlist:
+            wl = self.verify_whitelist(
+                self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
+            )
+            if self._allow_inactive:
+                pairlist = wl
+            else:
+                # Avoid implicit filtering of "verify_whitelist" to keep
+                # proper warnings in the log
+                pairlist = self._whitelist_for_active_markets(wl)
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._bt_pair_cache["pairlist"] = pairlist.copy()
+
+        return pairlist
 
     def filter_pairlist(self, pairlist: list[str], tickers: Tickers) -> list[str]:
         """

freqtrade/plugins/pairlistmanager.py

@@ -5,7 +5,7 @@ PairList manager class
 import logging
 from functools import partial
 
-from cachetools import TTLCache, cached
+from cachetools import LRUCache, TTLCache, cached
 
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
@@ -56,6 +56,7 @@ class PairListManager(LoggingMixin):
             )
 
         self._check_backtest()
+        self._not_expiring_cache: LRUCache = LRUCache(maxsize=1)
 
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
@@ -109,7 +110,15 @@ class PairListManager(LoggingMixin):
     @property
     def expanded_blacklist(self) -> list[str]:
         """The expanded blacklist (including wildcard expansion)"""
-        return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+        eblacklist = self._not_expiring_cache.get("eblacklist")
+
+        if not eblacklist:
+            eblacklist = expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._not_expiring_cache["eblacklist"] = eblacklist.copy()
+
+        return eblacklist
 
     @property
     def name_list(self) -> list[str]:
@@ -157,16 +166,17 @@ class PairListManager(LoggingMixin):
         :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
         :return: pairlist - blacklisted pairs
         """
-        try:
-            blacklist = self.expanded_blacklist
-        except ValueError as err:
-            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
-            return []
-        log_once = partial(self.log_once, logmethod=logmethod)
-        for pair in pairlist.copy():
-            if pair in blacklist:
-                log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
-                pairlist.remove(pair)
+        if self._blacklist:
+            try:
+                blacklist = self.expanded_blacklist
+            except ValueError as err:
+                logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+                return []
+            log_once = partial(self.log_once, logmethod=logmethod)
+            for pair in pairlist.copy():
+                if pair in blacklist:
+                    log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
+                    pairlist.remove(pair)
         return pairlist
 
     def verify_whitelist(