From fb86d8f8ff43646ded784c1ebace16cc1e8fd616 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 15:28:50 +0100 Subject: [PATCH 01/13] Add get_historic_ohlcv_as_df to support VolatilityFilter --- freqtrade/exchange/exchange.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2bbdb0d59..2f52c512f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -679,12 +679,25 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :returns List with candle (OHLCV) data + :return: List with candle (OHLCV) data """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms)) + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int) -> DataFrame: + """ + Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: OHLCV DataFrame + """ + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: From 109824c9a80cb78c7c4ec9d6f90cb1c8c3afa640 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 15:29:11 +0100 Subject: [PATCH 02/13] Add VolatilityFilter --- freqtrade/constants.py | 2 +- freqtrade/pairlist/AgeFilter.py | 2 +- freqtrade/pairlist/volatilityfilter.py | 89 ++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 freqtrade/pairlist/volatilityfilter.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3271dda39..55d802587 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -25,7 +25,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', - 'ShuffleFilter', 'SpreadFilter'] + 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/pairlist/AgeFilter.py index 19cf1c090..352fff082 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/pairlist/AgeFilter.py @@ -49,7 +49,7 @@ class AgeFilter(IPairList): return (f"{self.name} - Filtering pairs with age less than " f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, ticker: Dict) -> bool: """ Validate age for the ticker :param ticker: ticker dict as returned from ccxt.load_markets() diff --git a/freqtrade/pairlist/volatilityfilter.py b/freqtrade/pairlist/volatilityfilter.py new file mode 100644 index 000000000..6039f6f69 --- /dev/null +++ b/freqtrade/pairlist/volatilityfilter.py @@ -0,0 +1,89 @@ +""" +Minimum age (days listed) pair list filter +""" +import logging +from typing import Any, Dict + +import arrow +from cachetools.ttl import TTLCache + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class VolatilityFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('volatility_over_days', 10) + self._min_volatility = pairlistconfig.get('min_volatility', 0.01) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("VolatilityFilter requires min_days_listed to be >= 1") + if self._days > exchange.ohlcv_candle_limit: + raise OperationalException("VolatilityFilter requires min_days_listed to not exceed " + "exchange max request size " + f"({exchange.ohlcv_candle_limit})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with volatility below {self._min_volatility} " + f"over the last {plural(self._days, 'day')}.") + + def _validate_pair(self, ticker: Dict) -> bool: + """ + Validate volatility + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, False if it should be removed + """ + pair = ticker['symbol'] + # Check symbol in cache + if pair in self._pair_cache: + return self._pair_cache[pair] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days) + .float_timestamp) * 1000 + + daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair, + timeframe='1d', + since_ms=since_ms) + result = False + if daily_candles is not None: + highest_high = daily_candles['high'].max() + lowest_low = daily_candles['low'].min() + pct_change = (highest_high - lowest_low) / lowest_low + if pct_change >= self._min_volatility: + result = True + else: + self.log_on_refresh(logger.info, + f"Removed {pair} from whitelist, " + f"because volatility over {plural(self._days, 'day')} is " + f"{pct_change:.3f}, which is below the " + f"threshold of {self._min_volatility}.") + result = False + self._pair_cache[pair] = result + + return result From 191616e4e5cbb558f48ec39e67bf5399fbf87da5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 15:39:04 +0100 Subject: [PATCH 03/13] Add first tests for volatilityFilter --- tests/pairlist/test_pairlist.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 1f05bef1e..3e1cca30c 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -340,6 +340,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], "USDT", ['ETH/USDT', 'NANO/USDT']), + ([{"method": "StaticPairList"}, + {"method": "VolatilityFilter", "volatility_over_days": 10, + "min_volatility": 0.01, "refresh_period": 1440}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history_list, pairlists, base_currency, @@ -617,6 +621,11 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his None, "PriceFilter requires max_price to be >= 0" ), # OperationalException expected + ({"method": "VolatilityFilter", "volatility_over_days": 10, "min_volatility": 0.01}, + "[{'VolatilityFilter': 'VolatilityFilter - Filtering pairs with volatility below 0.01 " + "over the last days.'}]", + None + ), ]) def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, desc_expected, exception_expected): From 6b672cd0b95f8a35fe83dab95e6b931e6b85c51d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 15:43:29 +0100 Subject: [PATCH 04/13] Document volatilityFilter --- docs/includes/pairlists.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index e6a9fc1a8..301a5453d 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -19,6 +19,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`PriceFilter`](#pricefilter) * [`ShuffleFilter`](#shufflefilter) * [`SpreadFilter`](#spreadfilter) +* [`VolatilityFilter`](#volatilityfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. @@ -118,6 +119,27 @@ Example: If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. +#### VolatilityFilter + +Removes pairs where the difference between lowest low and highest high over `volatility_over_days` days is below `min_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. + +In the below example: +If volatility over the last 10 days is <1%, remove the pair from the whitelist. + +```json +"pairlists": [ + { + "method": "VolatilityFilter", + "volatility_over_days": 10, + "min_volatility": 0.01, + "refresh_period": 1440 + } +] +``` + +!!! Tip + This Filter can be used to automatically remove stable coin pairs, which have a very low volatility, and are therefore extremely hard to trade with profit. + ### Full example of Pairlist Handlers The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. @@ -137,6 +159,12 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "VolatilityFilter", + "volatility_over_days": 10, + "min_volatility": 0.01, + "refresh_period": 1440 + }, {"method": "ShuffleFilter", "seed": 42} ], ``` From f8fab5c4f8d120b7838cac24c6a0c7d30df85fc2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 15:51:39 +0100 Subject: [PATCH 05/13] Add tests for failure cases --- freqtrade/pairlist/volatilityfilter.py | 4 ++-- tests/pairlist/test_pairlist.py | 33 ++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/freqtrade/pairlist/volatilityfilter.py b/freqtrade/pairlist/volatilityfilter.py index 6039f6f69..e9ca61794 100644 --- a/freqtrade/pairlist/volatilityfilter.py +++ b/freqtrade/pairlist/volatilityfilter.py @@ -29,9 +29,9 @@ class VolatilityFilter(IPairList): self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) if self._days < 1: - raise OperationalException("VolatilityFilter requires min_days_listed to be >= 1") + raise OperationalException("VolatilityFilter requires volatility_over_days to be >= 1") if self._days > exchange.ohlcv_candle_limit: - raise OperationalException("VolatilityFilter requires min_days_listed to not exceed " + raise OperationalException("VolatilityFilter requires volatility_over_days to not exceed " "exchange max request size " f"({exchange.ohlcv_candle_limit})") diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 3e1cca30c..5bbb233b4 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -58,7 +58,7 @@ def whitelist_conf_2(default_conf): @pytest.fixture(scope="function") -def whitelist_conf_3(default_conf): +def whitelist_conf_agefilter(default_conf): default_conf['stake_currency'] = 'BTC' default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', @@ -532,7 +532,7 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf -def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers, caplog): +def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': -1}] @@ -547,7 +547,7 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) -def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers, caplog): +def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': 99999}] @@ -563,7 +563,7 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) -def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_history_list): +def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), @@ -575,7 +575,7 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), ) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_3) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 freqtrade.pairlists.refresh_pairlist() assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 @@ -586,6 +586,29 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_3, tickers, ohlcv_his assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count +def test_volatilityfilter_checks(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'VolatilityFilter', 'volatility_over_days': 99999}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'VolatilityFilter requires volatility_over_days to not exceed ' + r'exchange max request size \([0-9]+\)'): + get_patched_freqtradebot(mocker, default_conf) + + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'VolatilityFilter', 'volatility_over_days': 0}] + + with pytest.raises(OperationalException, + match='VolatilityFilter requires volatility_over_days to be >= 1'): + get_patched_freqtradebot(mocker, default_conf) + + @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, "max_price": 1.0}, From 2e1551a2ebce9cd9d288ba03a019778ff758b7ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 16:01:52 +0100 Subject: [PATCH 06/13] Improve tests of volatilityfilter --- docs/includes/pairlists.md | 2 +- freqtrade/pairlist/volatilityfilter.py | 4 ++-- tests/pairlist/test_pairlist.py | 33 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 301a5453d..7cd2369b1 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -138,7 +138,7 @@ If volatility over the last 10 days is <1%, remove the pair from the whitelist. ``` !!! Tip - This Filter can be used to automatically remove stable coin pairs, which have a very low volatility, and are therefore extremely hard to trade with profit. + This Filter can be used to automatically remove stable coin pairs, which have a very low volatility, and are therefore extremely difficult to trade with profit. ### Full example of Pairlist Handlers diff --git a/freqtrade/pairlist/volatilityfilter.py b/freqtrade/pairlist/volatilityfilter.py index e9ca61794..415b6e89e 100644 --- a/freqtrade/pairlist/volatilityfilter.py +++ b/freqtrade/pairlist/volatilityfilter.py @@ -31,8 +31,8 @@ class VolatilityFilter(IPairList): if self._days < 1: raise OperationalException("VolatilityFilter requires volatility_over_days to be >= 1") if self._days > exchange.ohlcv_candle_limit: - raise OperationalException("VolatilityFilter requires volatility_over_days to not exceed " - "exchange max request size " + raise OperationalException("VolatilityFilter requires volatility_over_days to not " + "exceed exchange max request size " f"({exchange.ohlcv_candle_limit})") @property diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 5bbb233b4..e9df5d3f4 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -609,6 +609,39 @@ def test_volatilityfilter_checks(mocker, default_conf, markets, tickers): get_patched_freqtradebot(mocker, default_conf) +@pytest.mark.parametrize('min_volatility,expected_length', [ + (0.01, 5), + (0.05, 0), # Setting volatility to 5% removes all pairs from the whitelist. +]) +def test_volatilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list, + min_volatility, expected_length): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'VolatilityFilter', 'volatility_over_days': 2, + 'min_volatility': min_volatility}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 + + previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + # Should not have increased since first call. + assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count + + @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, "max_price": 1.0}, From f12a8afd4151d6a2f069f5375291dc57e6b862b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 10:56:19 +0100 Subject: [PATCH 07/13] Add test for ohlcv_as_df --- tests/exchange/test_exchange.py | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e4452a83c..42681b367 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1307,6 +1307,57 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): assert log_has_re(r"Async code raised an exception: .*", caplog) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + ohlcv = [ + [ + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().shift(minutes=5).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().shift(minutes=10).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_candle_hist(pair, timeframe, since_ms): + return pair, timeframe, ohlcv + + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) + # one_call calculation * 1.8 should do 2 calls + + since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( + arrow.utcnow().int_timestamp - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above OHLCV data + assert len(ret) == 2 + assert isinstance(ret, DataFrame) + assert 'date' in ret.columns + assert 'open' in ret.columns + assert 'close' in ret.columns + assert 'high' in ret.columns + + def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ From 7e4fe23bf94128fa1df7477011d65d6d3ff2afd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 11:08:01 +0100 Subject: [PATCH 08/13] Add VolatilityFilter to full config --- config_full.json.example | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config_full.json.example b/config_full.json.example index 45c5c695c..0d82b9a2b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -67,7 +67,13 @@ {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, - {"method": "SpreadFilter", "max_spread_ratio": 0.005} + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "VolatilityFilter", + "volatility_over_days": 10, + "min_volatility": 0.01, + "refresh_period": 1440 + } ], "exchange": { "name": "bittrex", From 29c6a9263de13b3a480662d1c59b203512df8bd3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 15:50:44 +0100 Subject: [PATCH 09/13] Protect against 0 values --- freqtrade/pairlist/volatilityfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/volatilityfilter.py b/freqtrade/pairlist/volatilityfilter.py index 415b6e89e..14ac0c617 100644 --- a/freqtrade/pairlist/volatilityfilter.py +++ b/freqtrade/pairlist/volatilityfilter.py @@ -74,7 +74,7 @@ class VolatilityFilter(IPairList): if daily_candles is not None: highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() - pct_change = (highest_high - lowest_low) / lowest_low + pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 if pct_change >= self._min_volatility: result = True else: From 8f1d2ff0701bcf34609cec0c52e25697e3a8c65f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 19:47:27 +0100 Subject: [PATCH 10/13] Renamd volatilityFilter to RangeStabilityFilter --- config_full.json.example | 4 +-- docs/includes/pairlists.md | 22 ++++++------- freqtrade/constants.py | 2 +- ...ilityfilter.py => rangestabilityfilter.py} | 22 ++++++------- tests/pairlist/test_pairlist.py | 32 +++++++++---------- 5 files changed, 41 insertions(+), 41 deletions(-) rename freqtrade/pairlist/{volatilityfilter.py => rangestabilityfilter.py} (77%) diff --git a/config_full.json.example b/config_full.json.example index 0d82b9a2b..365b6180b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -69,8 +69,8 @@ {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, { - "method": "VolatilityFilter", - "volatility_over_days": 10, + "method": "RangeStabilityFilter", + "lookback_days": 10, "min_volatility": 0.01, "refresh_period": 1440 } diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 7cd2369b1..149e784bd 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -19,7 +19,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`PriceFilter`](#pricefilter) * [`ShuffleFilter`](#shufflefilter) * [`SpreadFilter`](#spreadfilter) -* [`VolatilityFilter`](#volatilityfilter) +* [`RangeStabilityFilter`](#rangestabilityfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. @@ -119,26 +119,26 @@ Example: If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. -#### VolatilityFilter +#### RangeStabilityFilter -Removes pairs where the difference between lowest low and highest high over `volatility_over_days` days is below `min_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. +Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. In the below example: -If volatility over the last 10 days is <1%, remove the pair from the whitelist. +If the trading range over the last 10 days is <1%, remove the pair from the whitelist. ```json "pairlists": [ { - "method": "VolatilityFilter", - "volatility_over_days": 10, - "min_volatility": 0.01, + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, "refresh_period": 1440 } ] ``` !!! Tip - This Filter can be used to automatically remove stable coin pairs, which have a very low volatility, and are therefore extremely difficult to trade with profit. + This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. ### Full example of Pairlist Handlers @@ -160,9 +160,9 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, {"method": "PriceFilter", "low_price_ratio": 0.01}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, { - "method": "VolatilityFilter", - "volatility_over_days": 10, - "min_volatility": 0.01, + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, "refresh_period": 1440 }, {"method": "ShuffleFilter", "seed": 42} diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 55d802587..2022556d2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -25,7 +25,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', - 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] + 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/pairlist/volatilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py similarity index 77% rename from freqtrade/pairlist/volatilityfilter.py rename to freqtrade/pairlist/rangestabilityfilter.py index 14ac0c617..f428bb113 100644 --- a/freqtrade/pairlist/volatilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -15,23 +15,23 @@ from freqtrade.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -class VolatilityFilter(IPairList): +class RangeStabilityFilter(IPairList): def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._days = pairlistconfig.get('volatility_over_days', 10) - self._min_volatility = pairlistconfig.get('min_volatility', 0.01) + self._days = pairlistconfig.get('lookback_days', 10) + self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._refresh_period = pairlistconfig.get('refresh_period', 1440) self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) if self._days < 1: - raise OperationalException("VolatilityFilter requires volatility_over_days to be >= 1") + raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") if self._days > exchange.ohlcv_candle_limit: - raise OperationalException("VolatilityFilter requires volatility_over_days to not " + raise OperationalException("RangeStabilityFilter requires lookback_days to not " "exceed exchange max request size " f"({exchange.ohlcv_candle_limit})") @@ -48,12 +48,12 @@ class VolatilityFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Filtering pairs with volatility below {self._min_volatility} " - f"over the last {plural(self._days, 'day')}.") + return (f"{self.name} - Filtering pairs with rate of change below " + f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.") def _validate_pair(self, ticker: Dict) -> bool: """ - Validate volatility + Validate trading range :param ticker: ticker dict as returned from ccxt.load_markets() :return: True if the pair can stay, False if it should be removed """ @@ -75,14 +75,14 @@ class VolatilityFilter(IPairList): highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 - if pct_change >= self._min_volatility: + if pct_change >= self._min_rate_of_change: result = True else: self.log_on_refresh(logger.info, f"Removed {pair} from whitelist, " - f"because volatility over {plural(self._days, 'day')} is " + f"because rate of change over {plural(self._days, 'day')} is " f"{pct_change:.3f}, which is below the " - f"threshold of {self._min_volatility}.") + f"threshold of {self._min_rate_of_change}.") result = False self._pair_cache[pair] = result diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index e9df5d3f4..d696e6d02 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -341,8 +341,8 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PriceFilter", "low_price_ratio": 0.02}], "USDT", ['ETH/USDT', 'NANO/USDT']), ([{"method": "StaticPairList"}, - {"method": "VolatilityFilter", "volatility_over_days": 10, - "min_volatility": 0.01, "refresh_period": 1440}], + {"method": "RangeStabilityFilter", "lookback_days": 10, + "min_rate_of_change": 0.01, "refresh_period": 1440}], "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, @@ -586,9 +586,9 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count -def test_volatilityfilter_checks(mocker, default_conf, markets, tickers): +def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, - {'method': 'VolatilityFilter', 'volatility_over_days': 99999}] + {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), @@ -597,27 +597,27 @@ def test_volatilityfilter_checks(mocker, default_conf, markets, tickers): ) with pytest.raises(OperationalException, - match=r'VolatilityFilter requires volatility_over_days to not exceed ' + match=r'RangeStabilityFilter requires lookback_days to not exceed ' r'exchange max request size \([0-9]+\)'): get_patched_freqtradebot(mocker, default_conf) default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, - {'method': 'VolatilityFilter', 'volatility_over_days': 0}] + {'method': 'RangeStabilityFilter', 'lookback_days': 0}] with pytest.raises(OperationalException, - match='VolatilityFilter requires volatility_over_days to be >= 1'): + match='RangeStabilityFilter requires lookback_days to be >= 1'): get_patched_freqtradebot(mocker, default_conf) -@pytest.mark.parametrize('min_volatility,expected_length', [ +@pytest.mark.parametrize('min_rate_of_change,expected_length', [ (0.01, 5), - (0.05, 0), # Setting volatility to 5% removes all pairs from the whitelist. + (0.05, 0), # Setting rate_of_change to 5% removes all pairs from the whitelist. ]) -def test_volatilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list, - min_volatility, expected_length): +def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list, + min_rate_of_change, expected_length): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, - {'method': 'VolatilityFilter', 'volatility_over_days': 2, - 'min_volatility': min_volatility}] + {'method': 'RangeStabilityFilter', 'lookback_days': 2, + 'min_rate_of_change': min_rate_of_change}] mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), @@ -677,9 +677,9 @@ def test_volatilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_ None, "PriceFilter requires max_price to be >= 0" ), # OperationalException expected - ({"method": "VolatilityFilter", "volatility_over_days": 10, "min_volatility": 0.01}, - "[{'VolatilityFilter': 'VolatilityFilter - Filtering pairs with volatility below 0.01 " - "over the last days.'}]", + ({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01}, + "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below " + "0.01 over the last days.'}]", None ), ]) From 0d349cb3550bfbbfa6af7915a53789074a76d6a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 19:59:18 +0100 Subject: [PATCH 11/13] Small finetuning --- config_full.json.example | 2 +- freqtrade/exchange/exchange.py | 2 +- freqtrade/pairlist/rangestabilityfilter.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 365b6180b..5ee2a1faf 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -71,7 +71,7 @@ { "method": "RangeStabilityFilter", "lookback_days": 10, - "min_volatility": 0.01, + "min_rate_of_change": 0.01, "refresh_period": 1440 } ], diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2f52c512f..18f4fbff5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -689,7 +689,7 @@ class Exchange: since_ms: int) -> DataFrame: """ Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe - :param pair: Pair to download + :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from :return: OHLCV DataFrame diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py index f428bb113..798d192bd 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -1,5 +1,5 @@ """ -Minimum age (days listed) pair list filter +Rate of change pairlist filter """ import logging from typing import Any, Dict From 8ae604d473df16f769c9cf43d4b37f3eec9f26cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Nov 2020 20:05:06 +0100 Subject: [PATCH 12/13] Ensure we're not running off of empty dataframes --- freqtrade/pairlist/rangestabilityfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py index 798d192bd..b460ff477 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -71,7 +71,7 @@ class RangeStabilityFilter(IPairList): timeframe='1d', since_ms=since_ms) result = False - if daily_candles is not None: + if daily_candles is not None and not daily_candles.empty: highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 From 6810192992df1e9f7943728c65c6c02e675c2d92 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Nov 2020 20:25:18 +0100 Subject: [PATCH 13/13] Update docstring for new filter --- freqtrade/pairlist/VolumePairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 44e5c52d7..7d3c2c653 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -49,7 +49,7 @@ class VolumePairList(IPairList): def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requires tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return True