Merge pull request #9861 from freqtrade/feat/sort_volatility

Add sorting to volatility and RangeStability pairlists

docs/includes/pairlists.md

@@ -450,6 +450,8 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting for this pairlist.
+
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
     Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time.
@@ -460,7 +462,7 @@ Volatility is the degree of historical variation of a pairs over time, it is mea
 
 This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 
-This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. 
+This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs.
 
 In the below example:
 If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h.
@@ -477,6 +479,8 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove
 ]
 ```
 
+Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist.
+
 ### Full example of Pairlist Handlers
 
 The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -3,7 +3,6 @@ Volatility pairlist filter
 """
 import logging
 import sys
-from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Dict, List, Optional
 
@@ -37,6 +36,7 @@ class VolatilityFilter(IPairList):
         self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
         self._refresh_period = pairlistconfig.get('refresh_period', 1440)
         self._def_candletype = self._config['candle_type_def']
+        self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
 
@@ -46,6 +46,9 @@ class VolatilityFilter(IPairList):
         if self._days > candle_limit:
             raise OperationalException("VolatilityFilter requires lookback_days to not "
                                        f"exceed exchange max request size ({candle_limit})")
+        if self._sort_direction not in [None, 'asc', 'desc']:
+            raise OperationalException("VolatilityFilter requires sort_direction to be "
+                                       "either None (undefined), 'asc' or 'desc'")
 
     @property
     def needstickers(self) -> bool:
@@ -89,6 +92,13 @@ class VolatilityFilter(IPairList):
                 "description": "Maximum Volatility",
                 "help": "Maximum volatility a pair must have to be considered.",
             },
+            "sort_direction": {
+                "type": "option",
+                "default": None,
+                "options": ["", "asc", "desc"],
+                "description": "Sort pairlist",
+                "help": "Sort Pairlist ascending or descending by volatility.",
+            },
             **IPairList.refresh_period_parameter()
         }
 
@@ -105,43 +115,61 @@ class VolatilityFilter(IPairList):
         since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days))
         candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
-        if self._enabled:
-            for p in deepcopy(pairlist):
-                daily_candles = candles[(p, '1d', self._def_candletype)] if (
-                    p, '1d', self._def_candletype) in candles else None
-                if not self._validate_pair_loc(p, daily_candles):
-                    pairlist.remove(p)
-        return pairlist
+        resulting_pairlist: List[str] = []
+        volatilitys: Dict[str, float] = {}
+        for p in pairlist:
+            daily_candles = candles.get((p, '1d', self._def_candletype), None)
 
-    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
-        """
-        Validate trading range
-        :param pair: Pair that's currently validated
-        :param daily_candles: Downloaded daily candles
-        :return: True if the pair can stay, false if it should be removed
-        """
+            volatility_avg = self._calculate_volatility(p, daily_candles)
+
+            if volatility_avg is not None:
+                if self._validate_pair_loc(p, volatility_avg):
+                    resulting_pairlist.append(p)
+                    volatilitys[p] = (
+                        volatility_avg if volatility_avg and not np.isnan(volatility_avg) else 0
+                    )
+            else:
+                self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info)
+
+        if self._sort_direction:
+            resulting_pairlist = sorted(resulting_pairlist,
+                                        key=lambda p: volatilitys[p],
+                                        reverse=self._sort_direction == 'desc')
+        return resulting_pairlist
+
+    def _calculate_volatility(self, pair: str,  daily_candles: DataFrame) -> Optional[float]:
         # Check symbol in cache
-        if (cached_res := self._pair_cache.get(pair, None)) is not None:
-            return cached_res
+        if (volatility_avg := self._pair_cache.get(pair, None)) is not None:
+            return volatility_avg
 
-        result = False
         if daily_candles is not None and not daily_candles.empty:
             returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"]))
             returns.fillna(0, inplace=True)
 
             volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days)
             volatility_avg = volatility_series.mean()
+            self._pair_cache[pair] = volatility_avg
 
-            if self._min_volatility <= volatility_avg <= self._max_volatility:
-                result = True
-            else:
-                self.log_once(f"Removed {pair} from whitelist, because volatility "
-                              f"over {self._days} {plural(self._days, 'day')} "
-                              f"is: {volatility_avg:.3f} "
-                              f"which is not in the configured range of "
-                              f"{self._min_volatility}-{self._max_volatility}.",
-                              logger.info)
-                result = False
-            self._pair_cache[pair] = result
+            return volatility_avg
+        else:
+            return None
 
+    def _validate_pair_loc(self, pair: str, volatility_avg: float) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param volatility_avg: Average volatility
+        :return: True if the pair can stay, false if it should be removed
+        """
+
+        if self._min_volatility <= volatility_avg <= self._max_volatility:
+            result = True
+        else:
+            self.log_once(f"Removed {pair} from whitelist, because volatility "
+                          f"over {self._days} {plural(self._days, 'day')} "
+                          f"is: {volatility_avg:.3f} "
+                          f"which is not in the configured range of "
+                          f"{self._min_volatility}-{self._max_volatility}.",
+                          logger.info)
+            result = False
         return result

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -2,7 +2,6 @@
 Rate of change pairlist filter
 """
 import logging
-from copy import deepcopy
 from datetime import timedelta
 from typing import Any, Dict, List, Optional
 
@@ -32,6 +31,7 @@ class RangeStabilityFilter(IPairList):
         self._max_rate_of_change = pairlistconfig.get('max_rate_of_change')
         self._refresh_period = pairlistconfig.get('refresh_period', 86400)
         self._def_candletype = self._config['candle_type_def']
+        self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None)
 
         self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
 
@@ -41,6 +41,9 @@ class RangeStabilityFilter(IPairList):
         if self._days > candle_limit:
             raise OperationalException("RangeStabilityFilter requires lookback_days to not "
                                        f"exceed exchange max request size ({candle_limit})")
+        if self._sort_direction not in [None, 'asc', 'desc']:
+            raise OperationalException("RangeStabilityFilter requires sort_direction to be "
+                                       "either None (undefined), 'asc' or 'desc'")
 
     @property
     def needstickers(self) -> bool:
@@ -87,6 +90,13 @@ class RangeStabilityFilter(IPairList):
                 "description": "Maximum Rate of Change",
                 "help": "Maximum rate of change to filter pairs.",
             },
+            "sort_direction": {
+                "type": "option",
+                "default": None,
+                "options": ["", "asc", "desc"],
+                "description": "Sort pairlist",
+                "help": "Sort Pairlist ascending or descending by rate of change.",
+            },
             **IPairList.refresh_period_parameter()
         }
 
@@ -103,45 +113,62 @@ class RangeStabilityFilter(IPairList):
         since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days + 1))
         candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms)
 
-        if self._enabled:
-            for p in deepcopy(pairlist):
-                daily_candles = candles[(p, '1d', self._def_candletype)] if (
-                    p, '1d', self._def_candletype) in candles else None
-                if not self._validate_pair_loc(p, daily_candles):
-                    pairlist.remove(p)
-        return pairlist
+        resulting_pairlist: List[str] = []
+        pct_changes: Dict[str, float] = {}
 
-    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
-        """
-        Validate trading range
-        :param pair: Pair that's currently validated
-        :param daily_candles: Downloaded daily candles
-        :return: True if the pair can stay, false if it should be removed
-        """
+        for p in pairlist:
+            daily_candles = candles.get((p, '1d', self._def_candletype), None)
+
+            pct_change = self._calculate_rate_of_change(p, daily_candles)
+
+            if pct_change is not None:
+                if self._validate_pair_loc(p, pct_change):
+                    resulting_pairlist.append(p)
+                    pct_changes[p] = pct_change
+            else:
+                self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info)
+
+        if self._sort_direction:
+            resulting_pairlist = sorted(resulting_pairlist,
+                                        key=lambda p: pct_changes[p],
+                                        reverse=self._sort_direction == 'desc')
+        return resulting_pairlist
+
+    def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> Optional[float]:
         # Check symbol in cache
-        if (cached_res := self._pair_cache.get(pair, None)) is not None:
-            return cached_res
-
-        result = True
+        if (pct_change := self._pair_cache.get(pair, None)) is not None:
+            return pct_change
         if daily_candles is not None and not daily_candles.empty:
+
             highest_high = daily_candles['high'].max()
             lowest_low = daily_candles['low'].min()
             pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0
-            if pct_change < self._min_rate_of_change:
-                self.log_once(f"Removed {pair} from whitelist, because rate of change "
-                              f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
-                              f"which is below the threshold of {self._min_rate_of_change}.",
-                              logger.info)
-                result = False
-            if self._max_rate_of_change:
-                if pct_change > self._max_rate_of_change:
-                    self.log_once(
-                        f"Removed {pair} from whitelist, because rate of change "
-                        f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
-                        f"which is above the threshold of {self._max_rate_of_change}.",
-                        logger.info)
-                    result = False
-            self._pair_cache[pair] = result
+            self._pair_cache[pair] = pct_change
+            return pct_change
         else:
-            self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
+            return None
+
+    def _validate_pair_loc(self, pair: str, pct_change: float) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param pct_change: Rate of change
+        :return: True if the pair can stay, false if it should be removed
+        """
+
+        result = True
+        if pct_change < self._min_rate_of_change:
+            self.log_once(f"Removed {pair} from whitelist, because rate of change "
+                          f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                          f"which is below the threshold of {self._min_rate_of_change}.",
+                          logger.info)
+            result = False
+        if self._max_rate_of_change:
+            if pct_change > self._max_rate_of_change:
+                self.log_once(
+                    f"Removed {pair} from whitelist, because rate of change "
+                    f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
+                    f"which is above the threshold of {self._max_rate_of_change}.",
+                    logger.info)
+                result = False
         return result

tests/conftest.py

@@ -142,8 +142,8 @@ def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days=
     return df
 
 
-def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
-    np.random.seed(42)
+def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42):
+    np.random.seed(random_seed)
 
     base = np.random.normal(20, 2, size=size)
     if timeframe == '1y':
@@ -174,9 +174,9 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
     return df
 
 
-def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05'):
+def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42):
     """ Generates data in the ohlcv format used by ccxt """
-    df = generate_test_data(timeframe, size, start)
+    df = generate_test_data(timeframe, size, start, random_seed)
     df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000
     return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns)))
 

tests/plugins/test_pairlist.py

@@ -19,7 +19,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist,
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.resolvers import PairListResolver
 from freqtrade.util.datetime_helpers import dt_now
-from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange,
+from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data, get_patched_exchange,
                             get_patched_freqtradebot, log_has, log_has_re, num_log_has)
 
 
@@ -748,6 +748,104 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
     assert log_has("PerformanceFilter is not available in this mode.", caplog)
 
 
+def test_VolatilityFilter_error(mocker, whitelist_conf) -> None:
+    volatility_filter = {"method": "VolatilityFilter", "lookback_days": -1}
+    whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter]
+
+    mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True))
+    exchange_mock = MagicMock()
+    exchange_mock.ohlcv_candle_limit = MagicMock(return_value=1000)
+
+    with pytest.raises(OperationalException,
+                       match=r"VolatilityFilter requires lookback_days to be >= 1*"):
+        PairListManager(exchange_mock, whitelist_conf, MagicMock())
+
+    volatility_filter = {"method": "VolatilityFilter", "lookback_days": 2000}
+    whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter]
+    with pytest.raises(OperationalException,
+                       match=r"VolatilityFilter requires lookback_days to not exceed exchange max"):
+        PairListManager(exchange_mock, whitelist_conf, MagicMock())
+
+    volatility_filter = {"method": "VolatilityFilter", "sort_direction": "Random"}
+    whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter]
+    with pytest.raises(OperationalException,
+                       match=r"VolatilityFilter requires sort_direction to be either "
+                             r"None .*'asc'.*'desc'"):
+        PairListManager(exchange_mock, whitelist_conf, MagicMock())
+
+
+@pytest.mark.parametrize('pairlist,expected_pairlist', [
+    ({"method": "VolatilityFilter", "sort_direction": "asc"},
+     ['XRP/BTC', 'ETH/BTC', 'LTC/BTC', 'TKN/BTC']),
+    ({"method": "VolatilityFilter", "sort_direction": "desc"},
+     ['TKN/BTC', 'LTC/BTC', 'ETH/BTC', 'XRP/BTC']),
+    ({"method": "VolatilityFilter", "sort_direction": "desc", 'min_volatility': 0.4},
+     ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']),
+    ({"method": "VolatilityFilter", "sort_direction": "asc", 'min_volatility': 0.4},
+     ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']),
+    ({"method": "VolatilityFilter", "sort_direction": "desc", 'max_volatility': 0.5},
+     ['LTC/BTC', 'ETH/BTC', 'XRP/BTC']),
+    ({"method": "VolatilityFilter", "sort_direction": "asc", 'max_volatility': 0.5},
+     ['XRP/BTC', 'ETH/BTC', 'LTC/BTC']),
+    ({"method": "RangeStabilityFilter", "sort_direction": "asc"},
+     ['ETH/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+    ({"method": "RangeStabilityFilter", "sort_direction": "desc"},
+     ['TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'ETH/BTC']),
+    ({"method": "RangeStabilityFilter", "sort_direction": "asc", 'min_rate_of_change': 0.4},
+     ['XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+    ({"method": "RangeStabilityFilter", "sort_direction": "desc", 'min_rate_of_change': 0.4},
+     ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+])
+def test_VolatilityFilter_RangeStabilityFilter_sort(
+        mocker, whitelist_conf, tickers, time_machine, pairlist, expected_pairlist) -> None:
+    whitelist_conf['pairlists'] = [
+        {'method': 'VolumePairList', 'number_assets': 10},
+        pairlist
+    ]
+
+    df1 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=42)
+    df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=2)
+    df3 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=3)
+    df4 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=4)
+    df5 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=5)
+    df6 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=6)
+
+    assert not df1.equals(df2)
+    time_machine.move_to('2022-01-15 00:00:00+00:00')
+
+    ohlcv_data = {
+        ('ETH/BTC', '1d', CandleType.SPOT): df1,
+        ('TKN/BTC', '1d', CandleType.SPOT): df2,
+        ('LTC/BTC', '1d', CandleType.SPOT): df3,
+        ('XRP/BTC', '1d', CandleType.SPOT): df4,
+        ('HOT/BTC', '1d', CandleType.SPOT): df5,
+        ('BLK/BTC', '1d', CandleType.SPOT): df6,
+
+    }
+    ohlcv_mock = MagicMock(return_value=ohlcv_data)
+    mocker.patch.multiple(
+        EXMS,
+        exchange_has=MagicMock(return_value=True),
+        refresh_latest_ohlcv=ohlcv_mock,
+        get_tickers=tickers
+
+    )
+
+    exchange = get_patched_exchange(mocker, whitelist_conf)
+    exchange.ohlcv_candle_limit = MagicMock(return_value=1000)
+    plm = PairListManager(exchange, whitelist_conf, MagicMock())
+
+    assert exchange.ohlcv_candle_limit.call_count == 2
+    plm.refresh_pairlist()
+    assert ohlcv_mock.call_count == 1
+    assert exchange.ohlcv_candle_limit.call_count == 2
+    assert plm.whitelist == expected_pairlist
+
+    plm.refresh_pairlist()
+    assert exchange.ohlcv_candle_limit.call_count == 2
+    assert ohlcv_mock.call_count == 1
+
+
 def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None:
     whitelist_conf['pairlists'] = [
         {"method": "StaticPairList"},
@@ -1095,6 +1193,13 @@ def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers):
                        match='RangeStabilityFilter requires lookback_days to be >= 1'):
         get_patched_freqtradebot(mocker, default_conf)
 
+    default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10},
+                                 {'method': 'RangeStabilityFilter', 'sort_direction': 'something'}]
+
+    with pytest.raises(OperationalException,
+                       match='RangeStabilityFilter requires sort_direction to be either None.*'):
+        get_patched_freqtradebot(mocker, default_conf)
+
 
 @pytest.mark.parametrize('min_rate_of_change,max_rate_of_change,expected_length', [
     (0.01, 0.99, 5),