From f98efdbe0787d9ea543d0589508f9511be2ea778 Mon Sep 17 00:00:00 2001 From: mrpabloyeah Date: Mon, 17 Nov 2025 13:09:48 +0100 Subject: [PATCH 1/3] Fix backtesting exception when no data is available for a pair --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9dc878fae..8a260cd76 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1425,7 +1425,7 @@ class Backtesting: # Row is treated as "current incomplete candle". # entry / exit signals are shifted by 1 to compensate for this. row = data[pair][row_index] - except IndexError: + except (IndexError, KeyError): # missing Data for one pair at the end. # Warnings for this are shown during data loading return None From 77e8a5357268054b7f2454c9018a104c1c252d2f Mon Sep 17 00:00:00 2001 From: mrpabloyeah Date: Mon, 24 Nov 2025 10:43:00 +0100 Subject: [PATCH 2/3] Fix backtesting exception when no data is available for a pair (new approach) --- freqtrade/optimize/backtesting.py | 11 +++++++---- freqtrade/plugins/pairlistmanager.py | 16 +++++++++++----- tests/optimize/test_backtesting.py | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8a260cd76..fd78162db 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -126,6 +126,7 @@ class Backtesting: self.config["dry_run"] = True self.price_pair_prec: dict[str, Series] = {} + self.available_pairs: list[str] = [] self.run_ids: dict[str, str] = {} self.strategylist: list[IStrategy] = [] self.all_bt_content: dict[str, BacktestContentType] = {} @@ -176,7 +177,8 @@ class Backtesting: self._validate_pairlists_for_backtesting() self.dataprovider.add_pairlisthandler(self.pairlists) - self.pairlists.refresh_pairlist() + self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False) + self.pairlists.refresh_pairlist(only_first=self.dynamic_pairlist) if len(self.pairlists.whitelist) == 0: raise OperationalException("No pair in whitelist.") @@ -211,7 +213,6 @@ class Backtesting: self._can_short = self.trading_mode != TradingMode.SPOT self._position_stacking: bool = self.config.get("position_stacking", False) self.enable_protections: bool = self.config.get("enable_protections", False) - self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False) migrate_data(config, self.exchange) self.init_backtest() @@ -335,10 +336,12 @@ class Backtesting: self.progress.set_new_value(1) self._load_bt_data_detail() self.price_pair_prec = {} + for pair in self.pairlists.whitelist: if pair in data: # Load price precision logic self.price_pair_prec[pair] = get_tick_size_over_time(data[pair]) + self.available_pairs.append(pair) return data, self.timerange def _load_bt_data_detail(self) -> None: @@ -1425,7 +1428,7 @@ class Backtesting: # Row is treated as "current incomplete candle". # entry / exit signals are shifted by 1 to compensate for this. row = data[pair][row_index] - except (IndexError, KeyError): + except IndexError: # missing Data for one pair at the end. # Warnings for this are shown during data loading return None @@ -1587,7 +1590,7 @@ class Backtesting: self.check_abort() if self.dynamic_pairlist and self.pairlists: - self.pairlists.refresh_pairlist() + self.pairlists.refresh_pairlist(pairs=self.available_pairs) pairs = self.pairlists.whitelist # Reset open trade count for this candle diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 9056d842e..f12ea795f 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -133,7 +133,7 @@ class PairListManager(LoggingMixin): def _get_cached_tickers(self) -> Tickers: return self._exchange.get_tickers() - def refresh_pairlist(self) -> None: + def refresh_pairlist(self, only_first: bool = False, pairs: list[str] | None = None) -> None: """Run pairlist through all configured Pairlist Handlers.""" # Tickers should be cached to avoid calling the exchange on each call. tickers: dict = {} @@ -143,10 +143,16 @@ class PairListManager(LoggingMixin): # Generate the pairlist with first Pairlist Handler in the chain pairlist = self._pairlist_handlers[0].gen_pairlist(tickers) - # Process all Pairlist Handlers in the chain - # except for the first one, which is the generator. - for pairlist_handler in self._pairlist_handlers[1:]: - pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) + if pairs: + for pair in pairlist: + if pair not in pairs: + pairlist.remove(pair) + + if not only_first: + # Process all Pairlist Handlers in the chain + # except for the first one, which is the generator. + for pairlist_handler in self._pairlist_handlers[1:]: + pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) # Validation against blacklist happens after the chain of Pairlist Handlers # to ensure blacklist is respected. diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 1c9373fb7..11b6cea24 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -2772,7 +2772,7 @@ def test_time_pair_generator_open_trades_first(mocker, default_conf, dynamic_pai dummy_row = (end_date, 1.0, 1.1, 0.9, 1.0, 0, 0, 0, 0, None, None) data = {pair: [dummy_row] for pair in pairs} - def mock_refresh(self): + def mock_refresh(self, **kwargs): # Simulate shuffle self._whitelist = pairs[::-1] # ['ETH/BTC', 'NEO/BTC', 'LTC/BTC', 'XRP/BTC'] From 72724037afe5afe98300e2f9b2fe5f616d15f08e Mon Sep 17 00:00:00 2001 From: mrpabloyeah Date: Wed, 26 Nov 2025 11:03:18 +0100 Subject: [PATCH 3/3] Replaced unsafe loop with a list comprehension & added docstring in refresh_pairlist() --- freqtrade/plugins/pairlistmanager.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index f12ea795f..7856ae15d 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -134,7 +134,19 @@ class PairListManager(LoggingMixin): return self._exchange.get_tickers() def refresh_pairlist(self, only_first: bool = False, pairs: list[str] | None = None) -> None: - """Run pairlist through all configured Pairlist Handlers.""" + """ + Run pairlist through all configured Pairlist Handlers. + + :param only_first: If True, only run the first PairList handler (the generator) + and skip all subsequent filters. Used during backtesting startup to ensure + historic data is loaded for the complete universe of pairs that the + generator can produce (even if later filters would reduce the list size). + Prevents missing data when a filter returns a variable number of pairs + across refresh cycles. + :param pairs: Optional list of pairs to intersect with the generated pairlist. + Only pairs present both in the generated list and this parameter are kept. + Used in backtesting to filter out pairs with no available data. + """ # Tickers should be cached to avoid calling the exchange on each call. tickers: dict = {} if self._tickers_needed: @@ -143,10 +155,9 @@ class PairListManager(LoggingMixin): # Generate the pairlist with first Pairlist Handler in the chain pairlist = self._pairlist_handlers[0].gen_pairlist(tickers) - if pairs: - for pair in pairlist: - if pair not in pairs: - pairlist.remove(pair) + # Optional intersection with an explicit list of pairs (used in backtesting) + if pairs is not None: + pairlist = [p for p in pairlist if p in pairs] if not only_first: # Process all Pairlist Handlers in the chain