Merge pull request #12214 from mrpabloyeah/fix-shufflefilter-behavior-in-backtesting

Fix ShuffleFilter behavior in backtesting

docs/commands/backtesting.md

@@ -10,6 +10,7 @@ usage: freqtrade backtesting [-h] [-v] [--no-color] [--logfile FILE] [-V]
                              [--stake-amount STAKE_AMOUNT] [--fee FLOAT]
                              [-p PAIRS [PAIRS ...]] [--eps]
                              [--enable-protections]
+                             [--enable-dynamic-pairlist]
                              [--dry-run-wallet DRY_RUN_WALLET]
                              [--timeframe-detail TIMEFRAME_DETAIL]
                              [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
@@ -44,9 +45,14 @@ options:
                         Allow buying the same pair multiple times (position
                         stacking).
   --enable-protections, --enableprotections
-                        Enable protections for backtesting.Will slow
+                        Enable protections for backtesting. Will slow
                         backtesting down by a considerable amount, but will
                         include configured protections
+  --enable-dynamic-pairlist
+                        Enables dynamic pairlist refreshes in backtesting. The
+                        pairlist will be generated for each new candle if
+                        you're using a pairlist handler that supports this
+                        feature, for example, ShuffleFilter.
   --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET
                         Starting balance, used for backtesting / hyperopt and
                         dry-runs.

docs/commands/hyperopt.md

@@ -44,7 +44,7 @@ options:
                         Allow buying the same pair multiple times (position
                         stacking).
   --enable-protections, --enableprotections
-                        Enable protections for backtesting.Will slow
+                        Enable protections for backtesting. Will slow
                         backtesting down by a considerable amount, but will
                         include configured protections
   --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET

docs/commands/lookahead-analysis.md

@@ -11,6 +11,7 @@ usage: freqtrade lookahead-analysis [-h] [-v] [--no-color] [--logfile FILE]
                                     [--stake-amount STAKE_AMOUNT]
                                     [--fee FLOAT] [-p PAIRS [PAIRS ...]]
                                     [--enable-protections]
+                                    [--enable-dynamic-pairlist]
                                     [--dry-run-wallet DRY_RUN_WALLET]
                                     [--timeframe-detail TIMEFRAME_DETAIL]
                                     [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
@@ -44,9 +45,14 @@ options:
                         Limit command to these pairs. Pairs are space-
                         separated.
   --enable-protections, --enableprotections
-                        Enable protections for backtesting.Will slow
+                        Enable protections for backtesting. Will slow
                         backtesting down by a considerable amount, but will
                         include configured protections
+  --enable-dynamic-pairlist
+                        Enables dynamic pairlist refreshes in backtesting. The
+                        pairlist will be generated for each new candle if
+                        you're using a pairlist handler that supports this
+                        feature, for example, ShuffleFilter.
   --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET
                         Starting balance, used for backtesting / hyperopt and
                         dry-runs.

freqtrade/commands/arguments.py

@@ -49,6 +49,7 @@ ARGS_BACKTEST = [
     *ARGS_COMMON_OPTIMIZE,
     "position_stacking",
     "enable_protections",
+    "enable_dynamic_pairlist",
     "dry_run_wallet",
     "timeframe_detail",
     "strategy_list",

freqtrade/commands/cli_options.py

@@ -184,12 +184,20 @@ AVAILABLE_CLI_OPTIONS = {
     "enable_protections": Arg(
         "--enable-protections",
         "--enableprotections",
-        help="Enable protections for backtesting."
+        help="Enable protections for backtesting. "
         "Will slow backtesting down by a considerable amount, but will include "
         "configured protections",
         action="store_true",
         default=False,
     ),
+    "enable_dynamic_pairlist": Arg(
+        "--enable-dynamic-pairlist",
+        help="Enables dynamic pairlist refreshes in backtesting. "
+        "The pairlist will be generated for each new candle if you're using a "
+        "pairlist handler that supports this feature, for example, ShuffleFilter.",
+        action="store_true",
+        default=False,
+    ),
     "strategy_list": Arg(
         "--strategy-list",
         help="Provide a space-separated list of strategies to backtest. "

freqtrade/configuration/configuration.py

@@ -259,7 +259,13 @@ class Configuration:
         self._args_to_config(
             config,
             argname="enable_protections",
-            logstring="Parameter --enable-protections detected, enabling Protections. ...",
+            logstring="Parameter --enable-protections detected, enabling Protections ...",
+        )
+
+        self._args_to_config(
+            config,
+            argname="enable_dynamic_pairlist",
+            logstring="Parameter --enable-dynamic-pairlist detected, enabling dynamic pairlist ...",
         )
 
         if self.args.get("max_open_trades"):

freqtrade/optimize/backtesting.py

@@ -211,6 +211,7 @@ class Backtesting:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get("position_stacking", False)
         self.enable_protections: bool = self.config.get("enable_protections", False)
+        self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
         migrate_data(config, self.exchange)
 
         self.init_backtest()
@@ -1584,6 +1585,11 @@ class Backtesting:
         for current_time in self._time_generator(start_date, end_date):
             # Loop for each main candle.
             self.check_abort()
+
+            if self.dynamic_pairlist and self.pairlists:
+                self.pairlists.refresh_pairlist()
+                pairs = self.pairlists.whitelist
+
             # Reset open trade count for this candle
             # Critical to avoid exceeding max_open_trades in backtesting
             # when timeframe-detail is used and trades close within the opening candle.

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -93,6 +93,8 @@ class ShuffleFilter(IPairList):
             return pairlist_new
         # Shuffle is done inplace
         self._random.shuffle(pairlist)
-        self.__pairlist_cache[pairlist_bef] = pairlist
+
+        if self._config.get("runmode") in (RunMode.LIVE, RunMode.DRY_RUN):
+            self.__pairlist_cache[pairlist_bef] = pairlist
 
         return pairlist

freqtrade/plugins/pairlist/StaticPairList.py

@@ -7,6 +7,9 @@ Provides pair white list as it configured in config
 import logging
 from copy import deepcopy
 
+from cachetools import LRUCache
+
+from freqtrade.enums import RunMode
 from freqtrade.exchange.exchange_types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
@@ -22,6 +25,8 @@ class StaticPairList(IPairList):
         super().__init__(*args, **kwargs)
 
         self._allow_inactive = self._pairlistconfig.get("allow_inactive", False)
+        # Pair cache - only used for optimize modes
+        self._bt_pair_cache: LRUCache = LRUCache(maxsize=1)
 
     @property
     def needstickers(self) -> bool:
@@ -60,15 +65,23 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: List of pairs
         """
-        wl = self.verify_whitelist(
-            self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
-        )
-        if self._allow_inactive:
-            return wl
-        else:
-            # Avoid implicit filtering of "verify_whitelist" to keep
-            # proper warnings in the log
-            return self._whitelist_for_active_markets(wl)
+        pairlist = self._bt_pair_cache.get("pairlist")
+
+        if not pairlist:
+            wl = self.verify_whitelist(
+                self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
+            )
+            if self._allow_inactive:
+                pairlist = wl
+            else:
+                # Avoid implicit filtering of "verify_whitelist" to keep
+                # proper warnings in the log
+                pairlist = self._whitelist_for_active_markets(wl)
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._bt_pair_cache["pairlist"] = pairlist.copy()
+
+        return pairlist
 
     def filter_pairlist(self, pairlist: list[str], tickers: Tickers) -> list[str]:
         """

freqtrade/plugins/pairlistmanager.py

@@ -5,7 +5,7 @@ PairList manager class
 import logging
 from functools import partial
 
-from cachetools import TTLCache, cached
+from cachetools import LRUCache, TTLCache, cached
 
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
@@ -56,6 +56,7 @@ class PairListManager(LoggingMixin):
             )
 
         self._check_backtest()
+        self._not_expiring_cache: LRUCache = LRUCache(maxsize=1)
 
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
@@ -109,7 +110,15 @@ class PairListManager(LoggingMixin):
     @property
     def expanded_blacklist(self) -> list[str]:
         """The expanded blacklist (including wildcard expansion)"""
-        return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+        eblacklist = self._not_expiring_cache.get("eblacklist")
+
+        if not eblacklist:
+            eblacklist = expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._not_expiring_cache["eblacklist"] = eblacklist.copy()
+
+        return eblacklist
 
     @property
     def name_list(self) -> list[str]:
@@ -157,16 +166,17 @@ class PairListManager(LoggingMixin):
         :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
         :return: pairlist - blacklisted pairs
         """
-        try:
-            blacklist = self.expanded_blacklist
-        except ValueError as err:
-            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
-            return []
-        log_once = partial(self.log_once, logmethod=logmethod)
-        for pair in pairlist.copy():
-            if pair in blacklist:
-                log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
-                pairlist.remove(pair)
+        if self._blacklist:
+            try:
+                blacklist = self.expanded_blacklist
+            except ValueError as err:
+                logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+                return []
+            log_once = partial(self.log_once, logmethod=logmethod)
+            for pair in pairlist.copy():
+                if pair in blacklist:
+                    log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
+                    pairlist.remove(pair)
         return pairlist
 
     def verify_whitelist(

tests/optimize/test_backtesting.py

@@ -27,7 +27,7 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename,
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import LocalTrade, Trade
 from freqtrade.resolvers import StrategyResolver
-from freqtrade.util.datetime_helpers import dt_utc
+from freqtrade.util import dt_now, dt_utc
 from tests.conftest import (
     CURRENT_TEST_STRATEGY,
     EXMS,
@@ -2715,3 +2715,75 @@ def test_get_backtest_metadata_filename():
     filename = "backtest_results_zip.zip"
     expected = Path("backtest_results_zip.meta.json")
     assert get_backtest_metadata_filename(filename) == expected
+
+
+@pytest.mark.parametrize("dynamic_pairlist", [True, False])
+def test_time_pair_generator_refresh_pairlist(mocker, default_conf, dynamic_pairlist):
+    patch_exchange(mocker)
+    default_conf["enable_dynamic_pairlist"] = dynamic_pairlist
+    backtesting = Backtesting(default_conf)
+    backtesting._set_strategy(backtesting.strategylist[0])
+    assert backtesting.dynamic_pairlist == dynamic_pairlist
+
+    refresh_mock = mocker.patch(
+        "freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist"
+    )
+
+    # Simulate 2 candles
+    start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC)
+    end_date = start_date + timedelta(minutes=10)
+    pairs = default_conf["exchange"]["pair_whitelist"]
+    data = {pair: [] for pair in pairs}
+
+    # Simulate backtest loop
+    list(backtesting.time_pair_generator(start_date, end_date, pairs, data))
+
+    if dynamic_pairlist:
+        assert refresh_mock.call_count == 2
+    else:
+        assert refresh_mock.call_count == 0
+
+
+@pytest.mark.parametrize("dynamic_pairlist", [True, False])
+def test_time_pair_generator_open_trades_first(mocker, default_conf, dynamic_pairlist):
+    patch_exchange(mocker)
+    default_conf["enable_dynamic_pairlist"] = dynamic_pairlist
+    backtesting = Backtesting(default_conf)
+    backtesting._set_strategy(backtesting.strategylist[0])
+    assert backtesting.dynamic_pairlist == dynamic_pairlist
+
+    pairs = ["XRP/BTC", "LTC/BTC", "NEO/BTC", "ETH/BTC"]
+
+    # Simulate open trades
+    trades = [
+        LocalTrade(pair="XRP/BTC", open_date=dt_now(), amount=1, open_rate=1),
+        LocalTrade(pair="NEO/BTC", open_date=dt_now(), amount=1, open_rate=1),
+    ]
+    LocalTrade.bt_trades_open = trades
+    LocalTrade.bt_trades_open_pp = {
+        "XRP/BTC": [trades[0]],
+        "NEO/BTC": [trades[1]],
+        "LTC/BTC": [],
+        "ETH/BTC": [],
+    }
+
+    start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC)
+    end_date = start_date + timedelta(minutes=5)
+    dummy_row = (end_date, 1.0, 1.1, 0.9, 1.0, 0, 0, 0, 0, None, None)
+    data = {pair: [dummy_row] for pair in pairs}
+
+    def mock_refresh(self):
+        # Simulate shuffle
+        self._whitelist = pairs[::-1]  # ['ETH/BTC', 'NEO/BTC', 'LTC/BTC', 'XRP/BTC']
+
+    mocker.patch("freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist", mock_refresh)
+
+    processed_pairs = []
+    for _, pair, _, _, _ in backtesting.time_pair_generator(start_date, end_date, pairs, data):
+        processed_pairs.append(pair)
+
+    # Open trades first in both cases
+    if dynamic_pairlist:
+        assert processed_pairs == ["XRP/BTC", "NEO/BTC", "ETH/BTC", "LTC/BTC"]
+    else:
+        assert processed_pairs == ["XRP/BTC", "NEO/BTC", "LTC/BTC", "ETH/BTC"]

tests/plugins/test_pairlist.py

@@ -1274,27 +1274,37 @@ def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None:
         {"method": "StaticPairList"},
         {"method": "ShuffleFilter", "seed": 43},
     ]
-    whitelist_conf["runmode"] = "backtest"
+    whitelist_conf["runmode"] = RunMode.BACKTEST
 
     exchange = get_patched_exchange(mocker, whitelist_conf)
     plm = PairListManager(exchange, whitelist_conf)
     assert log_has("Backtesting mode detected, applying seed value: 43", caplog)
 
+    plm.refresh_pairlist()
+    pl1 = deepcopy(plm.whitelist)
+    plm.refresh_pairlist()
+    assert plm.whitelist != pl1
+    assert set(plm.whitelist) == set(pl1)
+
+    caplog.clear()
+    whitelist_conf["runmode"] = RunMode.DRY_RUN
+    plm = PairListManager(exchange, whitelist_conf)
+    assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
+    assert log_has("Live mode detected, not applying seed.", caplog)
+
     with time_machine.travel("2021-09-01 05:01:00 +00:00") as t:
         plm.refresh_pairlist()
         pl1 = deepcopy(plm.whitelist)
         plm.refresh_pairlist()
         assert plm.whitelist == pl1
 
+        target = plm._pairlist_handlers[1]._random
+        shuffle_mock = mocker.patch.object(target, "shuffle", wraps=target.shuffle)
+
         t.shift(timedelta(minutes=10))
         plm.refresh_pairlist()
-        assert plm.whitelist != pl1
-
-    caplog.clear()
-    whitelist_conf["runmode"] = RunMode.DRY_RUN
-    plm = PairListManager(exchange, whitelist_conf)
-    assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
-    assert log_has("Live mode detected, not applying seed.", caplog)
+        assert shuffle_mock.call_count == 1
+        assert set(plm.whitelist) == set(pl1)
 
 
 @pytest.mark.usefixtures("init_persistence")