diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4d8894c26..17a829955 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -42,7 +42,7 @@ HYPEROPT_LOSS_BUILTIN = [ AVAILABLE_PAIRLISTS = [ "StaticPairList", "VolumePairList", - "PercentVolumeChangePairList", + "PercentChangePairList", "ProducerPairList", "RemotePairList", "MarketCapPairList", diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 62f0ca4de..34ca8a3e8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -121,6 +121,7 @@ class Exchange: # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency "ohlcv_volume_currency": "base", # "base" or "quote" "tickers_have_quoteVolume": True, + "tickers_have_percentage": True, "tickers_have_bid_ask": True, # bid / ask empty for fetch_tickers "tickers_have_price": True, "trades_pagination": "time", # Possible are "time" or "id" diff --git a/freqtrade/exchange/types.py b/freqtrade/exchange/types.py index 5568e4336..a0d315c78 100644 --- a/freqtrade/exchange/types.py +++ b/freqtrade/exchange/types.py @@ -12,6 +12,7 @@ class Ticker(TypedDict): last: Optional[float] quoteVolume: Optional[float] baseVolume: Optional[float] + percentage: Optional[float] # Several more - only listing required. diff --git a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py b/freqtrade/plugins/pairlist/PercentChangePairList.py similarity index 64% rename from freqtrade/plugins/pairlist/PercentVolumeChangePairList.py rename to freqtrade/plugins/pairlist/PercentChangePairList.py index 777acda60..a1be6729b 100644 --- a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py +++ b/freqtrade/plugins/pairlist/PercentChangePairList.py @@ -8,24 +8,24 @@ defined period import logging from datetime import timedelta -from typing import Any, Dict, List, Literal +from typing import Any, Dict, List, Literal, Optional from cachetools import TTLCache from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date -from freqtrade.exchange.types import Tickers +from freqtrade.exchange.types import Ticker, Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) -SORT_VALUES = ["rolling_volume_change"] +SORT_VALUES = ["percentage"] -class PercentVolumeChangePairList(IPairList): +class PercentChangePairList(IPairList): is_pairlist_generator = True supports_backtesting = SupportsBacktesting.NO @@ -50,6 +50,7 @@ class PercentVolumeChangePairList(IPairList): self._lookback_days = self._pairlistconfig.get("lookback_days", 0) self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d") self._lookback_period = self._pairlistconfig.get("lookback_period", 0) + self._sort_direction: Optional[str] = self._pairlistconfig.get("sort_direction", "desc") self._def_candletype = self._config["candle_type_def"] if (self._lookback_days > 0) & (self._lookback_period > 0): @@ -80,11 +81,11 @@ class PercentVolumeChangePairList(IPairList): if not self._use_range and not ( self._exchange.exchange_has("fetchTickers") - and self._exchange.get_option("tickers_have_change") + and self._exchange.get_option("tickers_have_percentage") ): raise OperationalException( "Exchange does not support dynamic whitelist in this configuration. " - "Please edit your config and either remove PercentVolumeChangePairList, " + "Please edit your config and either remove PercentChangePairList, " "or switch to using candles. and restart the bot." ) @@ -94,9 +95,7 @@ class PercentVolumeChangePairList(IPairList): candle_limit = self._exchange.ohlcv_candle_limit( self._lookback_timeframe, self._config["candle_type_def"] ) - if self._lookback_period < 4: - raise OperationalException("ChangeFilter requires lookback_period to be >= 4") - self.log_once(f"Candle limit is {candle_limit}", logger.info) + if self._lookback_period > candle_limit: raise OperationalException( "ChangeFilter requires lookback_period to not " @@ -119,14 +118,11 @@ class PercentVolumeChangePairList(IPairList): """ Short whitelist method description - used for startup-messages """ - return ( - f"{self.name} - top {self._pairlistconfig['number_assets']} percent " - f"volume change pairs." - ) + return f"{self.name} - top {self._pairlistconfig['number_assets']} percent change pairs." @staticmethod def description() -> str: - return "Provides dynamic pair list based on percentage volume change." + return "Provides dynamic pair list based on percentage change." @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: @@ -156,12 +152,14 @@ class PercentVolumeChangePairList(IPairList): "description": "Maximum value", "help": "Maximum value to use for filtering the pairlist.", }, - "refresh_period": { - "type": "number", - "default": 1800, - "description": "Refresh period", - "help": "Refresh period in seconds", + "sort_direction": { + "type": "option", + "default": "desc", + "options": ["", "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist ascending or descending by rate of change.", }, + **IPairList.refresh_period_parameter(), "lookback_days": { "type": "number", "default": 0, @@ -233,83 +231,24 @@ class PercentVolumeChangePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning) + filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] if self._use_range: - filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] - - # get lookback period in ms, for exchange ohlcv fetch - since_ms = ( - int( - timeframe_to_prev_date( - self._lookback_timeframe, - dt_now() - + timedelta( - minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min - ), - ).timestamp() - ) - * 1000 - ) - - to_ms = ( - int( - timeframe_to_prev_date( - self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) - ).timestamp() - ) - * 1000 - ) - - # todo: utc date output for starting date - self.log_once( - f"Using change range of {self._lookback_period} candles, timeframe: " - f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " - f"till {format_ms_time(to_ms)}", - logger.info, - ) - needed_pairs: ListPairsWithTimeframes = [ - (p, self._lookback_timeframe, self._def_candletype) - for p in [s["symbol"] for s in filtered_tickers] - if p not in self._pair_cache - ] - - candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) - - for i, p in enumerate(filtered_tickers): - pair_candles = ( - candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] - if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles - else None - ) - - # in case of candle data calculate typical price and change for candle - if pair_candles is not None and not pair_candles.empty: - pair_candles["rolling_volume_sum"] = ( - pair_candles["volume"].rolling(window=self._lookback_period).sum() - ) - pair_candles["rolling_volume_change"] = ( - pair_candles["rolling_volume_sum"].pct_change() * 100 - ) - - # ensure that a rolling sum over the lookback_period is built - # if pair_candles contains more candles than lookback_period - rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1] - - # replace change with a range change sum calculated above - filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change - self.log_once(f"ticker {filtered_tickers[i]}", logger.info) - else: - filtered_tickers[i]["rolling_volume_change"] = 0 + # calculating using lookback_period + self.fetch_percent_change_from_lookback_period(filtered_tickers) else: - filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # Fetching 24h change by default from supported exchange tickers + self.fetch_percent_change_from_tickers(filtered_tickers, tickers) filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value] if self._max_value is not None: filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value] - sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) + sorted_tickers = sorted( + filtered_tickers, + reverse=self._sort_direction == "desc", + key=lambda t: t[self._sort_key], + ) - self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info) # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers]) pairs = self.verify_blacklist(pairs, logmethod=logger.info) @@ -317,3 +256,91 @@ class PercentVolumeChangePairList(IPairList): pairs = pairs[: self._number_pairs] return pairs + + def fetch_candles_for_lookback_period(self, filtered_tickers): + since_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, + dt_now() + + timedelta( + minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min + ), + ).timestamp() + ) + * 1000 + ) + to_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) + ).timestamp() + ) + * 1000 + ) + # todo: utc date output for starting date + self.log_once( + f"Using change range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", + logger.info, + ) + needed_pairs: ListPairsWithTimeframes = [ + (p, self._lookback_timeframe, self._def_candletype) + for p in [s["symbol"] for s in filtered_tickers] + if p not in self._pair_cache + ] + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) + return candles + + def fetch_percent_change_from_lookback_period(self, filtered_tickers): + # get lookback period in ms, for exchange ohlcv fetch + candles = self.fetch_candles_for_lookback_period(filtered_tickers) + + for i, p in enumerate(filtered_tickers): + pair_candles = ( + candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] + if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles + else None + ) + + # in case of candle data calculate typical price and change for candle + if pair_candles is not None and not pair_candles.empty: + current_close = pair_candles["close"].iloc[-1] + previous_close = pair_candles["close"].shift(self._lookback_period).iloc[-1] + pct_change = ( + ((current_close - previous_close) / previous_close) if previous_close > 0 else 0 + ) + + # replace change with a range change sum calculated above + filtered_tickers[i]["percentage"] = pct_change + self.log_once(f"Tickers: {filtered_tickers}", logger.info) + else: + filtered_tickers[i]["percentage"] = 0 + + def fetch_percent_change_from_tickers(self, filtered_tickers, tickers): + for i, p in enumerate(filtered_tickers): + # Filter out assets + if not self._validate_pair( + p["symbol"], tickers[p["symbol"]] if p["symbol"] in tickers else None + ): + filtered_tickers.remove(p) + else: + filtered_tickers[i]["percentage"] = tickers[p["symbol"]]["percentage"] + + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: + """ + Check if one price-step (pip) is > than a certain barrier. + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.fetch_ticker + :return: True if the pair can stay, false if it should be removed + """ + if not ticker or "percentage" not in ticker or ticker["percentage"] is None: + self.log_once( + f"Removed {pair} from whitelist, because " + "ticker['percentage'] is empty (Usually no trade in the last 24h).", + logger.info, + ) + return False + + return True diff --git a/tests/conftest.py b/tests/conftest.py index fee8cab72..22bc2556e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2187,7 +2187,7 @@ def tickers(): "first": None, "last": 530.21, "change": 0.558, - "percentage": None, + "percentage": 2.349, "average": None, "baseVolume": 72300.0659, "quoteVolume": 37670097.3022171, diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 7e5fd4c12..31e746d5c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -31,11 +31,11 @@ from tests.conftest import ( ) -# Exclude RemotePairList and PercentVolumeChangePairList from tests. +# Exclude RemotePairList and PercentVolumePairList from tests. # They have mandatory parameters, and requires special handling, -# which happens in test_remotepairlist and test_percentvolumechangepairlist. +# which happens in test_remotepairlist and test_percentchangepairlist. TESTABLE_PAIRLISTS = [ - p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentVolumeChangePairList"] + p for p in AVAILABLE_PAIRLISTS if p not in ["RemotePairList", "PercentChangePairList"] ] diff --git a/tests/plugins/test_percentvolumechangepairlist.py b/tests/plugins/test_percentchangepairlist.py similarity index 82% rename from tests/plugins/test_percentvolumechangepairlist.py rename to tests/plugins/test_percentchangepairlist.py index b09307151..0a7960d22 100644 --- a/tests/plugins/test_percentvolumechangepairlist.py +++ b/tests/plugins/test_percentchangepairlist.py @@ -7,7 +7,7 @@ import pytest from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException -from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList +from freqtrade.plugins.pairlist.PercentChangePairList import PercentChangePairList from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import ( EXMS, @@ -33,9 +33,9 @@ def rpl_config(default_conf): def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, } @@ -44,7 +44,7 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): with pytest.raises( OperationalException, match=r"Exchange does not support dynamic whitelist in this configuration. " - r"Please edit your config and either remove PercentVolumeChangePairList, " + r"Please edit your config and either remove PercentChangePairList, " r"or switch to using candles. and restart the bot.", ): get_patched_freqtradebot(mocker, rpl_config) @@ -53,9 +53,9 @@ def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 1800, "lookback_days": 4, @@ -71,12 +71,31 @@ def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): get_patched_freqtradebot(mocker, rpl_config) +def test_volume_change_pair_list_init_invalid_sort_key(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentChangePairList", + "number_assets": 2, + "sort_key": "wrong_key", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1, + } + ] + + with pytest.raises( + OperationalException, + match=r"key wrong_key not in \['percentage'\]", + ): + get_patched_freqtradebot(mocker, rpl_config) + + def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 3, @@ -95,41 +114,9 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", - "min_value": 0, - "refresh_period": 86400, - "lookback_days": 3, - } - ] - - with pytest.raises( - OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" - ): - get_patched_freqtradebot(mocker, rpl_config) - - rpl_config["pairlists"] = [ - { - "method": "PercentVolumeChangePairList", - "number_assets": 2, - "sort_key": "rolling_volume_change", - "min_value": 0, - "refresh_period": 86400, - "lookback_period": 3, - } - ] - - with pytest.raises( - OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" - ): - get_patched_freqtradebot(mocker, rpl_config) - - rpl_config["pairlists"] = [ - { - "method": "PercentVolumeChangePairList", - "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 1001, @@ -147,8 +134,8 @@ def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", - "sort_key": "rolling_volume_change", + "method": "PercentChangePairList", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, } @@ -165,9 +152,9 @@ def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, "lookback_days": 4, @@ -210,7 +197,7 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic ) ), ("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame( - # Make sure always have highest rolling_volume_change + # Make sure always have highest percentage { "timestamp": [ "2024-07-01 00:00:00", @@ -234,24 +221,25 @@ def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tic exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) result = remote_pairlist.gen_pairlist(tickers) assert len(result) == 2 - assert result == ["TKN/USDT", "BTC/USDT"] + assert result == ["NEO/USDT", "TKN/USDT"] def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "refresh_period": 86400, + "sort_direction": "asc", "lookback_days": 4, } ] @@ -272,7 +260,7 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], + "close": [101, 102, 103, 104, 105, 105], "volume": [1000, 1500, 2000, 2500, 3000, 3500], } ), @@ -289,8 +277,8 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], - "volume": [1000, 1500, 2000, 2500, 3000, 3500], + "close": [101, 102, 103, 104, 105, 104], + "volume": [1000, 1500, 2000, 2500, 3000, 3400], } ), } @@ -299,22 +287,22 @@ def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_mac exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) assert len(result) == 2 - assert result == ["ETH/USDT", "XRP/USDT"] + assert result == ["XRP/USDT", "ETH/USDT"] def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine): rpl_config["pairlists"] = [ { - "method": "PercentVolumeChangePairList", + "method": "PercentChangePairList", "number_assets": 2, - "sort_key": "rolling_volume_change", + "sort_key": "percentage", "min_value": 0, "max_value": 15, "refresh_period": 86400, @@ -356,7 +344,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma "open": [100, 102, 101, 103, 104, 105], "high": [102, 103, 102, 104, 105, 106], "low": [99, 101, 100, 102, 103, 104], - "close": [101, 102, 103, 104, 105, 106], + "close": [101, 102, 103, 104, 105, 101], "volume": [1000, 1500, 2000, 2500, 3000, 3500], } ), @@ -366,7 +354,7 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") pairlistmanager = PairListManager(exchange, rpl_config) - remote_pairlist = PercentVolumeChangePairList( + remote_pairlist = PercentChangePairList( exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 ) @@ -374,3 +362,28 @@ def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_ma assert len(result) == 1 assert result == ["ETH/USDT"] + + +def test_gen_pairlist_from_tickers(mocker, rpl_config, tickers): + rpl_config["pairlists"] = [ + { + "method": "PercentChangePairList", + "number_assets": 2, + "sort_key": "percentage", + "min_value": 0, + } + ] + + mocker.patch(f"{EXMS}.exchange_has", MagicMock(return_value=True)) + + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.gen_pairlist(tickers.return_value) + + assert len(result) == 1 + assert result == ["ETH/USDT"]