Merge pull request #10352 from freqtrade/feat/pairlist_bt_check

Feat/pairlist_bt_check

freqtrade/optimize/backtesting.py

@@ -217,8 +217,6 @@ class Backtesting:
             raise OperationalException(
                 "VolumePairList not allowed for backtesting. Please use StaticPairList instead."
             )
-        if "PerformanceFilter" in self.pairlists.name_list:
-            raise OperationalException("PerformanceFilter not allowed for backtesting.")
 
         if len(self.strategylist) > 1 and "PrecisionFilter" in self.pairlists.name_list:
             raise OperationalException(

freqtrade/plugins/pairlist/AgeFilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class AgeFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -7,13 +7,15 @@ from typing import List
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class FullTradesFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     @property
     def needstickers(self) -> bool:
         """

freqtrade/plugins/pairlist/IPairList.py

@@ -5,6 +5,7 @@ PairList Handler base class
 import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
 
 from freqtrade.constants import Config
@@ -51,8 +52,20 @@ PairlistParameter = Union[
 ]
 
 
+class SupportsBacktesting(str, Enum):
+    """
+    Enum to indicate if a Pairlist Handler supports backtesting.
+    """
+
+    YES = "yes"
+    NO = "no"
+    NO_ACTION = "no_action"
+    BIASED = "biased"
+
+
 class IPairList(LoggingMixin, ABC):
     is_pairlist_generator = False
+    supports_backtesting: SupportsBacktesting = SupportsBacktesting.NO
 
     def __init__(
         self,

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -11,7 +11,7 @@ from cachetools import TTLCache
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.coin_gecko import FtCoinGeckoApi
 
 
@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
 
 class MarketCapPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/OffsetFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, List
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class OffsetFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -9,13 +9,15 @@ import pandas as pd
 
 from freqtrade.exchange.types import Tickers
 from freqtrade.persistence import Trade
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PerformanceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO_ACTION
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PrecisionFilter.py

@@ -8,13 +8,15 @@ from typing import Optional
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import ROUND_UP
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PrecisionFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/PriceFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class PriceFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.BIASED
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/ProducerPairList.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ class ProducerPairList(IPairList):
     """
 
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/RemotePairList.py

@@ -16,7 +16,7 @@ from freqtrade import __version__
 from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -25,6 +25,8 @@ logger = logging.getLogger(__name__)
 
 class RemotePairList(IPairList):
     is_pairlist_generator = True
+    # Potential winner bias
+    supports_backtesting = SupportsBacktesting.BIASED
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -9,7 +9,7 @@ from typing import Dict, List, Literal
 from freqtrade.enums import RunMode
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util.periodic_cache import PeriodicCache
 
 
@@ -19,6 +19,8 @@ ShuffleValues = Literal["candle", "iteration"]
 
 
 class ShuffleFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.YES
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -7,13 +7,15 @@ from typing import Dict, Optional
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Ticker
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
 
 
 class SpreadFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/StaticPairList.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from typing import Dict, List
 
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
 
 logger = logging.getLogger(__name__)
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.YES
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -15,7 +15,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -27,6 +27,8 @@ class VolatilityFilter(IPairList):
     Filters pairs by volatility
     """
 
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlist/VolumePairList.py

@@ -14,7 +14,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
 from freqtrade.exchange.types import Tickers
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_now, format_ms_time
 
 
@@ -26,6 +26,7 @@ SORT_VALUES = ["quoteVolume"]
 
 class VolumePairList(IPairList):
     is_pairlist_generator = True
+    supports_backtesting = SupportsBacktesting.NO
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -13,7 +13,7 @@ from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.misc import plural
-from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
+from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 from freqtrade.util import dt_floor_day, dt_now, dt_ts
 
 
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class RangeStabilityFilter(IPairList):
+    supports_backtesting = SupportsBacktesting.NO
+
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 

freqtrade/plugins/pairlistmanager.py

@@ -11,10 +11,11 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType
+from freqtrade.enums.runmode import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange.types import Tickers
 from freqtrade.mixins import LoggingMixin
-from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.IPairList import IPairList, SupportsBacktesting
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver
 
@@ -57,9 +58,44 @@ class PairListManager(LoggingMixin):
                 f"{invalid}."
             )
 
+        self._check_backtest()
+
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
 
+    def _check_backtest(self) -> None:
+        if self._config["runmode"] not in (RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT):
+            return
+
+        pairlist_errors: List[str] = []
+        noaction_pairlists: List[str] = []
+        biased_pairlists: List[str] = []
+        for pairlist_handler in self._pairlist_handlers:
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO:
+                pairlist_errors.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.NO_ACTION:
+                noaction_pairlists.append(pairlist_handler.name)
+            if pairlist_handler.supports_backtesting == SupportsBacktesting.BIASED:
+                biased_pairlists.append(pairlist_handler.name)
+
+        if noaction_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(noaction_pairlists)} do not generate "
+                "any changes during backtesting. While it's safe to leave them enabled, they will "
+                "not behave like in dry/live modes. "
+            )
+
+        if biased_pairlists:
+            logger.warning(
+                f"Pairlist Handlers {', '.join(biased_pairlists)} will introduce a lookahead bias "
+                "to your backtest results, as they use today's data - which inheritly suffers from "
+                "'winner bias'."
+            )
+        if pairlist_errors:
+            raise OperationalException(
+                f"Pairlist Handlers {', '.join(pairlist_errors)} do not support backtesting."
+            )
+
     @property
     def whitelist(self) -> List[str]:
         """The current whitelist"""

tests/optimize/test_backtesting.py

@@ -429,7 +429,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
         backtesting.start()
 
 
-def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None:
+def test_backtesting_no_pair_left(default_conf, mocker) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(
         "freqtrade.data.history.history_utils.load_pair_history",
@@ -449,13 +449,6 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
     with pytest.raises(OperationalException, match="No pair in whitelist."):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "VolumePairList", "number_assets": 5}]
-    with pytest.raises(
-        OperationalException,
-        match=r"VolumePairList not allowed for backtesting\..*StaticPairList.*",
-    ):
-        Backtesting(default_conf)
-
     default_conf.update(
         {
             "pairlists": [{"method": "StaticPairList"}],
@@ -469,7 +462,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) ->
         Backtesting(default_conf)
 
 
-def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None:
+def test_backtesting_pairlist_list(default_conf, mocker, tickers) -> None:
     mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True))
     mocker.patch(f"{EXMS}.get_tickers", tickers)
     mocker.patch(f"{EXMS}.price_to_precision", lambda s, x, y: y)
@@ -495,12 +488,6 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti
     ):
         Backtesting(default_conf)
 
-    default_conf["pairlists"] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}]
-    with pytest.raises(
-        OperationalException, match="PerformanceFilter not allowed for backtesting."
-    ):
-        Backtesting(default_conf)
-
     default_conf["pairlists"] = [
         {"method": "StaticPairList"},
         {"method": "PrecisionFilter"},

tests/plugins/test_pairlist.py

@@ -38,6 +38,7 @@ TESTABLE_PAIRLISTS = [p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairLis
 
 @pytest.fixture(scope="function")
 def whitelist_conf(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -68,6 +69,7 @@ def whitelist_conf(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_2(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -94,6 +96,7 @@ def whitelist_conf_2(default_conf):
 
 @pytest.fixture(scope="function")
 def whitelist_conf_agefilter(default_conf):
+    default_conf["runmode"] = "dry_run"
     default_conf["stake_currency"] = "BTC"
     default_conf["exchange"]["pair_whitelist"] = [
         "ETH/BTC",
@@ -773,7 +776,7 @@ def test_VolumePairList_whitelist_gen(
     whitelist_result,
     caplog,
 ) -> None:
-    whitelist_conf["runmode"] = "backtest"
+    whitelist_conf["runmode"] = "util_exchange"
     whitelist_conf["pairlists"] = pairlists
     whitelist_conf["stake_currency"] = base_currency
 
@@ -2387,3 +2390,65 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
         OperationalException, match="This filter only support marketcap rank up to 250."
     ):
         PairListManager(exchange, default_conf_usdt)
+
+
+@pytest.mark.parametrize(
+    "pairlists,expected_error,expected_warning",
+    [
+        (
+            [{"method": "StaticPairList"}],
+            None,  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "VolumePairList", "number_assets": 10}],
+            "VolumePairList",  # Error
+            None,  # Warning
+        ),
+        (
+            [{"method": "MarketCapPairList", "number_assets": 10}],
+            None,  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+        (
+            [{"method": "StaticPairList"}, {"method": "FullTradesFilter"}],
+            None,  # Error
+            r"FullTradesFilter do not generate.*",  # Warning
+        ),
+        (  # combi, fails and warns
+            [
+                {"method": "VolumePairList", "number_assets": 10},
+                {"method": "MarketCapPairList", "number_assets": 10},
+            ],
+            "VolumePairList",  # Error
+            r"MarketCapPairList.*lookahead.*",  # Warning
+        ),
+    ],
+)
+def test_backtesting_modes(
+    mocker, default_conf_usdt, pairlists, expected_error, expected_warning, caplog, markets, tickers
+):
+    default_conf_usdt["runmode"] = "dry_run"
+    default_conf_usdt["pairlists"] = pairlists
+
+    mocker.patch.multiple(
+        EXMS,
+        markets=PropertyMock(return_value=markets),
+        exchange_has=MagicMock(return_value=True),
+        get_tickers=tickers,
+    )
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+
+    # Dry run mode - works always
+    PairListManager(exchange, default_conf_usdt)
+
+    default_conf_usdt["runmode"] = "backtest"
+    if expected_error:
+        with pytest.raises(OperationalException, match=f"Pairlist Handlers {expected_error}.*"):
+            PairListManager(exchange, default_conf_usdt)
+
+    if not expected_error:
+        PairListManager(exchange, default_conf_usdt)
+
+    if expected_warning:
+        assert log_has_re(f"Pairlist Handlers {expected_warning}", caplog)