Merge branch 'freqtrade:develop' into delist

freqtrade/commands/arguments.py

@@ -49,6 +49,7 @@ ARGS_BACKTEST = [
     *ARGS_COMMON_OPTIMIZE,
     "position_stacking",
     "enable_protections",
+    "enable_dynamic_pairlist",
     "dry_run_wallet",
     "timeframe_detail",
     "strategy_list",

freqtrade/commands/cli_options.py

@@ -184,12 +184,20 @@ AVAILABLE_CLI_OPTIONS = {
     "enable_protections": Arg(
         "--enable-protections",
         "--enableprotections",
-        help="Enable protections for backtesting."
+        help="Enable protections for backtesting. "
         "Will slow backtesting down by a considerable amount, but will include "
         "configured protections",
         action="store_true",
         default=False,
     ),
+    "enable_dynamic_pairlist": Arg(
+        "--enable-dynamic-pairlist",
+        help="Enables dynamic pairlist refreshes in backtesting. "
+        "The pairlist will be generated for each new candle if you're using a "
+        "pairlist handler that supports this feature, for example, ShuffleFilter.",
+        action="store_true",
+        default=False,
+    ),
     "strategy_list": Arg(
         "--strategy-list",
         help="Provide a space-separated list of strategies to backtest. "

freqtrade/configuration/configuration.py

@@ -259,7 +259,13 @@ class Configuration:
         self._args_to_config(
             config,
             argname="enable_protections",
-            logstring="Parameter --enable-protections detected, enabling Protections. ...",
+            logstring="Parameter --enable-protections detected, enabling Protections ...",
+        )
+
+        self._args_to_config(
+            config,
+            argname="enable_dynamic_pairlist",
+            logstring="Parameter --enable-dynamic-pairlist detected, enabling dynamic pairlist ...",
         )
 
         if self.args.get("max_open_trades"):

freqtrade/data/converter/converter.py

@@ -181,7 +181,6 @@ def trim_dataframes(
 
 def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     """
-    TODO: This should get a dedicated test
     Gets order book list, returns dataframe with below format per suggested by creslin
     -------------------------------------------------------------------
      b_sum       b_size       bids       asks       a_size       a_sum

freqtrade/exchange/exchange.py

@@ -692,12 +692,13 @@ class Exchange:
             # Reload async markets, then assign them to sync api
             retrier(self._load_async_markets, retries=retries)(reload=True)
             self._markets = self._api_async.markets
-            self._api.set_markets(self._api_async.markets, self._api_async.currencies)
+            self._api.set_markets_from_exchange(self._api_async)
             # Assign options array, as it contains some temporary information from the exchange.
+            # TODO: investigate with ccxt if it's safe to remove `.options`
             self._api.options = self._api_async.options
             if self._exchange_ws:
                 # Set markets to avoid reloading on websocket api
-                self._ws_async.set_markets(self._api.markets, self._api.currencies)
+                self._ws_async.set_markets_from_exchange(self._api_async)
                 self._ws_async.options = self._api.options
             self._last_markets_refresh = dt_ts()
 

freqtrade/optimize/backtesting.py

@@ -211,6 +211,7 @@ class Backtesting:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get("position_stacking", False)
         self.enable_protections: bool = self.config.get("enable_protections", False)
+        self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
         migrate_data(config, self.exchange)
 
         self.init_backtest()
@@ -1584,6 +1585,11 @@ class Backtesting:
         for current_time in self._time_generator(start_date, end_date):
             # Loop for each main candle.
             self.check_abort()
+
+            if self.dynamic_pairlist and self.pairlists:
+                self.pairlists.refresh_pairlist()
+                pairs = self.pairlists.whitelist
+
             # Reset open trade count for this candle
             # Critical to avoid exceeding max_open_trades in backtesting
             # when timeframe-detail is used and trades close within the opening candle.

freqtrade/plugins/pairlist/ShuffleFilter.py

@@ -93,6 +93,8 @@ class ShuffleFilter(IPairList):
             return pairlist_new
         # Shuffle is done inplace
         self._random.shuffle(pairlist)
-        self.__pairlist_cache[pairlist_bef] = pairlist
+
+        if self._config.get("runmode") in (RunMode.LIVE, RunMode.DRY_RUN):
+            self.__pairlist_cache[pairlist_bef] = pairlist
 
         return pairlist

freqtrade/plugins/pairlist/StaticPairList.py

@@ -7,6 +7,9 @@ Provides pair white list as it configured in config
 import logging
 from copy import deepcopy
 
+from cachetools import LRUCache
+
+from freqtrade.enums import RunMode
 from freqtrade.exchange.exchange_types import Tickers
 from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
 
@@ -22,6 +25,8 @@ class StaticPairList(IPairList):
         super().__init__(*args, **kwargs)
 
         self._allow_inactive = self._pairlistconfig.get("allow_inactive", False)
+        # Pair cache - only used for optimize modes
+        self._bt_pair_cache: LRUCache = LRUCache(maxsize=1)
 
     @property
     def needstickers(self) -> bool:
@@ -60,15 +65,23 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers). May be cached.
         :return: List of pairs
         """
-        wl = self.verify_whitelist(
-            self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
-        )
-        if self._allow_inactive:
-            return wl
-        else:
-            # Avoid implicit filtering of "verify_whitelist" to keep
-            # proper warnings in the log
-            return self._whitelist_for_active_markets(wl)
+        pairlist = self._bt_pair_cache.get("pairlist")
+
+        if not pairlist:
+            wl = self.verify_whitelist(
+                self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
+            )
+            if self._allow_inactive:
+                pairlist = wl
+            else:
+                # Avoid implicit filtering of "verify_whitelist" to keep
+                # proper warnings in the log
+                pairlist = self._whitelist_for_active_markets(wl)
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._bt_pair_cache["pairlist"] = pairlist.copy()
+
+        return pairlist
 
     def filter_pairlist(self, pairlist: list[str], tickers: Tickers) -> list[str]:
         """

freqtrade/plugins/pairlistmanager.py

@@ -5,7 +5,7 @@ PairList manager class
 import logging
 from functools import partial
 
-from cachetools import TTLCache, cached
+from cachetools import LRUCache, TTLCache, cached
 
 from freqtrade.constants import Config, ListPairsWithTimeframes
 from freqtrade.data.dataprovider import DataProvider
@@ -56,6 +56,7 @@ class PairListManager(LoggingMixin):
             )
 
         self._check_backtest()
+        self._not_expiring_cache: LRUCache = LRUCache(maxsize=1)
 
         refresh_period = config.get("pairlist_refresh_period", 3600)
         LoggingMixin.__init__(self, logger, refresh_period)
@@ -109,7 +110,15 @@ class PairListManager(LoggingMixin):
     @property
     def expanded_blacklist(self) -> list[str]:
         """The expanded blacklist (including wildcard expansion)"""
-        return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+        eblacklist = self._not_expiring_cache.get("eblacklist")
+
+        if not eblacklist:
+            eblacklist = expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+
+            if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
+                self._not_expiring_cache["eblacklist"] = eblacklist.copy()
+
+        return eblacklist
 
     @property
     def name_list(self) -> list[str]:
@@ -157,16 +166,17 @@ class PairListManager(LoggingMixin):
         :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
         :return: pairlist - blacklisted pairs
         """
-        try:
-            blacklist = self.expanded_blacklist
-        except ValueError as err:
-            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
-            return []
-        log_once = partial(self.log_once, logmethod=logmethod)
-        for pair in pairlist.copy():
-            if pair in blacklist:
-                log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
-                pairlist.remove(pair)
+        if self._blacklist:
+            try:
+                blacklist = self.expanded_blacklist
+            except ValueError as err:
+                logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+                return []
+            log_once = partial(self.log_once, logmethod=logmethod)
+            for pair in pairlist.copy():
+                if pair in blacklist:
+                    log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
+                    pairlist.remove(pair)
         return pairlist
 
     def verify_whitelist(

freqtrade/rpc/telegram.py

@@ -360,11 +360,9 @@ class Telegram(RPCHandler):
                 await asyncio.sleep(2)
         if self._app.updater:
             await self._app.updater.start_polling(
-                bootstrap_retries=-1,
+                bootstrap_retries=10,
                 timeout=20,
-                # read_latency=60,  # Assumed transmission latency
                 drop_pending_updates=True,
-                # stop_signals=[],  # Necessary as we don't run on the main thread
             )
             while True:
                 await asyncio.sleep(10)