Merge pull request #9187 from stash86/full-pairlist

Add FullTradesPairlist

config_examples/config_full.example.json

@@ -70,6 +70,7 @@
     },
     "pairlists": [
         {"method": "StaticPairList"},
+        {"method": "FullTradesFilter"},
         {
             "method": "VolumePairList",
             "number_assets": 20,

docs/includes/pairlists.md

@@ -25,6 +25,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`ProducerPairList`](#producerpairlist)
 * [`RemotePairList`](#remotepairlist)
 * [`AgeFilter`](#agefilter)
+* [`FullTradesFilter`](#fulltradesfilter)
 * [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
 * [`PrecisionFilter`](#precisionfilter)
@@ -236,6 +237,17 @@ be caught out buying before the pair has finished dropping in price.
 
 This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
 
+#### FullTradesFilter
+
+Shrink whitelist to consist only in-trade pairs when the trade slots are full (when `max_open_trades` isn't being set to `-1` in the config).
+
+When the trade slots are full, there is no need to calculate indicators of the rest of the pairs (except informative pairs) since no new trade can be opened. By shrinking the whitelist to just the in-trade pairs, you can improve calculation speeds and reduce CPU usage. When a trade slot is free (either a trade is closed or `max_open_trades` value in config is increased), then the whitelist will return to normal state.
+
+When multiple pairlist filters are being used, it's recommended to put this filter at second position directly below the primary pairlist, so when the trade slots are full, the bot don't have to download data for the rest of the filters.
+
+!!! Warning "Backtesting"
+    `FullTradesFilter` does not support backtesting mode.
+
 #### OffsetFilter
 
 Offsets an incoming pairlist by a given `offset` value.

freqtrade/constants.py

@@ -33,7 +33,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
+                       'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',

freqtrade/plugins/pairlist/FullTradesFilter.py

@@ -0,0 +1,57 @@
+"""
+Full trade slots pair list filter
+"""
+import logging
+from typing import Any, Dict, List
+
+from freqtrade.constants import Config
+from freqtrade.exchange.types import Tickers
+from freqtrade.persistence import Trade
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class FullTradesFilter(IPairList):
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Config, pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short allowlist method description - used for startup-messages
+        """
+        return f"{self.name} - Shrink whitelist when trade slots are full."
+
+    @staticmethod
+    def description() -> str:
+        return "Shrink whitelist when trade slots are full."
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]:
+        """
+        Filters and sorts pairlist and returns the allowlist again.
+        Called on each bot iteration - please use internal caching if necessary
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers). May be cached.
+        :return: new allowlist
+        """
+        # Get the number of open trades and max open trades config
+        num_open = Trade.get_open_trade_count()
+        max_trades = self._config['max_open_trades']
+
+        if (num_open >= max_trades) and (max_trades > 0):
+            return []
+
+        return pairlist

tests/plugins/test_pairlist.py

@@ -14,7 +14,7 @@ from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import CandleType, RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.persistence import Trade
+from freqtrade.persistence import LocalTrade, Trade
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.resolvers import PairListResolver
@@ -1463,3 +1463,53 @@ def test_ProducerPairlist(mocker, whitelist_conf, markets):
     pm.refresh_pairlist()
     assert len(pm.whitelist) == 4
     assert pm.whitelist == ['TKN/BTC'] + pairs
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
+    default_conf_usdt['exchange']['pair_whitelist'].extend(['ADA/USDT', 'XRP/USDT', 'ETC/USDT'])
+    default_conf_usdt['pairlists'] = [
+        {"method": "StaticPairList"},
+        {"method": "FullTradesFilter"}
+    ]
+    default_conf_usdt['max_open_trades'] = -1
+    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
+    exchange = get_patched_exchange(mocker, default_conf_usdt)
+    pm = PairListManager(exchange, default_conf_usdt)
+    pm.refresh_pairlist()
+
+    assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']
+
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+        create_mock_trades_usdt(fee)
+        pm.refresh_pairlist()
+
+        # Unlimited max open trades, so no change to whitelist
+        pm.refresh_pairlist()
+        assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']
+
+        # Set max_open_trades to 4, the filter should empty the whitelist
+        default_conf_usdt['max_open_trades'] = 4
+        pm.refresh_pairlist()
+        assert pm.whitelist == []
+        assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog)
+
+        list_trades = LocalTrade.get_open_trades()
+        assert len(list_trades) == 4
+
+        # Move to 1 hour later, close a trade, so original sorting is restored.
+        t.move_to("2021-09-01 07:00:00 +00:00")
+        list_trades[2].close(12)
+        Trade.commit()
+
+        # open trades count below max_open_trades, whitelist restored
+        list_trades = LocalTrade.get_open_trades()
+        assert len(list_trades) == 3
+        pm.refresh_pairlist()
+        assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT']
+
+        # Set max_open_trades to 3, the filter should empty the whitelist
+        default_conf_usdt['max_open_trades'] = 3
+        pm.refresh_pairlist()
+        assert pm.whitelist == []
+        assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog)