diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index d1dd2cda7..960f2d210 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -450,6 +450,8 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from ] ``` +Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting for this pairlist. + !!! Tip This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time. @@ -460,7 +462,7 @@ Volatility is the degree of historical variation of a pairs over time, it is mea This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. -This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. +This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. In the below example: If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h. @@ -477,6 +479,8 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove ] ``` +Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist. + ### Full example of Pairlist Handlers The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value. diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index ef72486e1..cdd171e91 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -3,7 +3,6 @@ Volatility pairlist filter """ import logging import sys -from copy import deepcopy from datetime import timedelta from typing import Any, Dict, List, Optional @@ -37,6 +36,7 @@ class VolatilityFilter(IPairList): self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) self._refresh_period = pairlistconfig.get('refresh_period', 1440) self._def_candletype = self._config['candle_type_def'] + self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None) self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -46,6 +46,9 @@ class VolatilityFilter(IPairList): if self._days > candle_limit: raise OperationalException("VolatilityFilter requires lookback_days to not " f"exceed exchange max request size ({candle_limit})") + if self._sort_direction not in [None, 'asc', 'desc']: + raise OperationalException("VolatilityFilter requires sort_direction to be " + "either None (undefined), 'asc' or 'desc'") @property def needstickers(self) -> bool: @@ -89,6 +92,13 @@ class VolatilityFilter(IPairList): "description": "Maximum Volatility", "help": "Maximum volatility a pair must have to be considered.", }, + "sort_direction": { + "type": "option", + "default": None, + "options": ["", "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist ascending or descending by volatility.", + }, **IPairList.refresh_period_parameter() } @@ -105,43 +115,61 @@ class VolatilityFilter(IPairList): since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days)) candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) - if self._enabled: - for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._def_candletype)] if ( - p, '1d', self._def_candletype) in candles else None - if not self._validate_pair_loc(p, daily_candles): - pairlist.remove(p) - return pairlist + resulting_pairlist: List[str] = [] + volatilitys: Dict[str, float] = {} + for p in pairlist: + daily_candles = candles.get((p, '1d', self._def_candletype), None) - def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: - """ - Validate trading range - :param pair: Pair that's currently validated - :param daily_candles: Downloaded daily candles - :return: True if the pair can stay, false if it should be removed - """ + volatility_avg = self._calculate_volatility(p, daily_candles) + + if volatility_avg is not None: + if self._validate_pair_loc(p, volatility_avg): + resulting_pairlist.append(p) + volatilitys[p] = ( + volatility_avg if volatility_avg and not np.isnan(volatility_avg) else 0 + ) + else: + self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) + + if self._sort_direction: + resulting_pairlist = sorted(resulting_pairlist, + key=lambda p: volatilitys[p], + reverse=self._sort_direction == 'desc') + return resulting_pairlist + + def _calculate_volatility(self, pair: str, daily_candles: DataFrame) -> Optional[float]: # Check symbol in cache - if (cached_res := self._pair_cache.get(pair, None)) is not None: - return cached_res + if (volatility_avg := self._pair_cache.get(pair, None)) is not None: + return volatility_avg - result = False if daily_candles is not None and not daily_candles.empty: returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) returns.fillna(0, inplace=True) volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) volatility_avg = volatility_series.mean() + self._pair_cache[pair] = volatility_avg - if self._min_volatility <= volatility_avg <= self._max_volatility: - result = True - else: - self.log_once(f"Removed {pair} from whitelist, because volatility " - f"over {self._days} {plural(self._days, 'day')} " - f"is: {volatility_avg:.3f} " - f"which is not in the configured range of " - f"{self._min_volatility}-{self._max_volatility}.", - logger.info) - result = False - self._pair_cache[pair] = result + return volatility_avg + else: + return None + def _validate_pair_loc(self, pair: str, volatility_avg: float) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param volatility_avg: Average volatility + :return: True if the pair can stay, false if it should be removed + """ + + if self._min_volatility <= volatility_avg <= self._max_volatility: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because volatility " + f"over {self._days} {plural(self._days, 'day')} " + f"is: {volatility_avg:.3f} " + f"which is not in the configured range of " + f"{self._min_volatility}-{self._max_volatility}.", + logger.info) + result = False return result diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 49fba59b9..0480f60d0 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -2,7 +2,6 @@ Rate of change pairlist filter """ import logging -from copy import deepcopy from datetime import timedelta from typing import Any, Dict, List, Optional @@ -32,6 +31,7 @@ class RangeStabilityFilter(IPairList): self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') self._refresh_period = pairlistconfig.get('refresh_period', 86400) self._def_candletype = self._config['candle_type_def'] + self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None) self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -41,6 +41,9 @@ class RangeStabilityFilter(IPairList): if self._days > candle_limit: raise OperationalException("RangeStabilityFilter requires lookback_days to not " f"exceed exchange max request size ({candle_limit})") + if self._sort_direction not in [None, 'asc', 'desc']: + raise OperationalException("RangeStabilityFilter requires sort_direction to be " + "either None (undefined), 'asc' or 'desc'") @property def needstickers(self) -> bool: @@ -87,6 +90,13 @@ class RangeStabilityFilter(IPairList): "description": "Maximum Rate of Change", "help": "Maximum rate of change to filter pairs.", }, + "sort_direction": { + "type": "option", + "default": None, + "options": ["", "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist ascending or descending by rate of change.", + }, **IPairList.refresh_period_parameter() } @@ -103,45 +113,62 @@ class RangeStabilityFilter(IPairList): since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days + 1)) candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) - if self._enabled: - for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._def_candletype)] if ( - p, '1d', self._def_candletype) in candles else None - if not self._validate_pair_loc(p, daily_candles): - pairlist.remove(p) - return pairlist + resulting_pairlist: List[str] = [] + pct_changes: Dict[str, float] = {} - def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: - """ - Validate trading range - :param pair: Pair that's currently validated - :param daily_candles: Downloaded daily candles - :return: True if the pair can stay, false if it should be removed - """ + for p in pairlist: + daily_candles = candles.get((p, '1d', self._def_candletype), None) + + pct_change = self._calculate_rate_of_change(p, daily_candles) + + if pct_change is not None: + if self._validate_pair_loc(p, pct_change): + resulting_pairlist.append(p) + pct_changes[p] = pct_change + else: + self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) + + if self._sort_direction: + resulting_pairlist = sorted(resulting_pairlist, + key=lambda p: pct_changes[p], + reverse=self._sort_direction == 'desc') + return resulting_pairlist + + def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> Optional[float]: # Check symbol in cache - if (cached_res := self._pair_cache.get(pair, None)) is not None: - return cached_res - - result = True + if (pct_change := self._pair_cache.get(pair, None)) is not None: + return pct_change if daily_candles is not None and not daily_candles.empty: + highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 - if pct_change < self._min_rate_of_change: - self.log_once(f"Removed {pair} from whitelist, because rate of change " - f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " - f"which is below the threshold of {self._min_rate_of_change}.", - logger.info) - result = False - if self._max_rate_of_change: - if pct_change > self._max_rate_of_change: - self.log_once( - f"Removed {pair} from whitelist, because rate of change " - f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " - f"which is above the threshold of {self._max_rate_of_change}.", - logger.info) - result = False - self._pair_cache[pair] = result + self._pair_cache[pair] = pct_change + return pct_change else: - self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info) + return None + + def _validate_pair_loc(self, pair: str, pct_change: float) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param pct_change: Rate of change + :return: True if the pair can stay, false if it should be removed + """ + + result = True + if pct_change < self._min_rate_of_change: + self.log_once(f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is below the threshold of {self._min_rate_of_change}.", + logger.info) + result = False + if self._max_rate_of_change: + if pct_change > self._max_rate_of_change: + self.log_once( + f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is above the threshold of {self._max_rate_of_change}.", + logger.info) + result = False return result diff --git a/tests/conftest.py b/tests/conftest.py index a322bfd77..d894a7908 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,8 +142,8 @@ def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days= return df -def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): - np.random.seed(42) +def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42): + np.random.seed(random_seed) base = np.random.normal(20, 2, size=size) if timeframe == '1y': @@ -174,9 +174,9 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): return df -def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05'): +def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42): """ Generates data in the ohlcv format used by ccxt """ - df = generate_test_data(timeframe, size, start) + df = generate_test_data(timeframe, size, start, random_seed) df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000 return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns))) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d125f8896..57affc731 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -19,7 +19,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from freqtrade.util.datetime_helpers import dt_now -from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange, +from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data, get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re, num_log_has) @@ -748,6 +748,104 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +def test_VolatilityFilter_error(mocker, whitelist_conf) -> None: + volatility_filter = {"method": "VolatilityFilter", "lookback_days": -1} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + exchange_mock = MagicMock() + exchange_mock.ohlcv_candle_limit = MagicMock(return_value=1000) + + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires lookback_days to be >= 1*"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + volatility_filter = {"method": "VolatilityFilter", "lookback_days": 2000} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires lookback_days to not exceed exchange max"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + volatility_filter = {"method": "VolatilityFilter", "sort_direction": "Random"} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires sort_direction to be either " + r"None .*'asc'.*'desc'"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + +@pytest.mark.parametrize('pairlist,expected_pairlist', [ + ({"method": "VolatilityFilter", "sort_direction": "asc"}, + ['XRP/BTC', 'ETH/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc"}, + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC', 'XRP/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc", 'min_volatility': 0.4}, + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "asc", 'min_volatility': 0.4}, + ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc", 'max_volatility': 0.5}, + ['LTC/BTC', 'ETH/BTC', 'XRP/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "asc", 'max_volatility': 0.5}, + ['XRP/BTC', 'ETH/BTC', 'LTC/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "asc"}, + ['ETH/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "desc"}, + ['TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'ETH/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "asc", 'min_rate_of_change': 0.4}, + ['XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "desc", 'min_rate_of_change': 0.4}, + ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']), +]) +def test_VolatilityFilter_RangeStabilityFilter_sort( + mocker, whitelist_conf, tickers, time_machine, pairlist, expected_pairlist) -> None: + whitelist_conf['pairlists'] = [ + {'method': 'VolumePairList', 'number_assets': 10}, + pairlist + ] + + df1 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=42) + df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=2) + df3 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=3) + df4 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=4) + df5 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=5) + df6 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=6) + + assert not df1.equals(df2) + time_machine.move_to('2022-01-15 00:00:00+00:00') + + ohlcv_data = { + ('ETH/BTC', '1d', CandleType.SPOT): df1, + ('TKN/BTC', '1d', CandleType.SPOT): df2, + ('LTC/BTC', '1d', CandleType.SPOT): df3, + ('XRP/BTC', '1d', CandleType.SPOT): df4, + ('HOT/BTC', '1d', CandleType.SPOT): df5, + ('BLK/BTC', '1d', CandleType.SPOT): df6, + + } + ohlcv_mock = MagicMock(return_value=ohlcv_data) + mocker.patch.multiple( + EXMS, + exchange_has=MagicMock(return_value=True), + refresh_latest_ohlcv=ohlcv_mock, + get_tickers=tickers + + ) + + exchange = get_patched_exchange(mocker, whitelist_conf) + exchange.ohlcv_candle_limit = MagicMock(return_value=1000) + plm = PairListManager(exchange, whitelist_conf, MagicMock()) + + assert exchange.ohlcv_candle_limit.call_count == 2 + plm.refresh_pairlist() + assert ohlcv_mock.call_count == 1 + assert exchange.ohlcv_candle_limit.call_count == 2 + assert plm.whitelist == expected_pairlist + + plm.refresh_pairlist() + assert exchange.ohlcv_candle_limit.call_count == 2 + assert ohlcv_mock.call_count == 1 + + def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: whitelist_conf['pairlists'] = [ {"method": "StaticPairList"}, @@ -1095,6 +1193,13 @@ def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): match='RangeStabilityFilter requires lookback_days to be >= 1'): get_patched_freqtradebot(mocker, default_conf) + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'sort_direction': 'something'}] + + with pytest.raises(OperationalException, + match='RangeStabilityFilter requires sort_direction to be either None.*'): + get_patched_freqtradebot(mocker, default_conf) + @pytest.mark.parametrize('min_rate_of_change,max_rate_of_change,expected_length', [ (0.01, 0.99, 5),