From 4b1177e07e48007da7721984a9eb25c482f83a93 Mon Sep 17 00:00:00 2001 From: jainanuj94 Date: Wed, 24 Jul 2024 19:09:45 +0530 Subject: [PATCH] 10348 | Create new pair list to dynamically fetch pairs based on percent volume change --- .../pairlist/PercentVolumeChangePairList.py | 316 +++++++++++++++ .../test_percentvolumechangepairlist.py | 376 ++++++++++++++++++ 2 files changed, 692 insertions(+) create mode 100644 freqtrade/plugins/pairlist/PercentVolumeChangePairList.py create mode 100644 tests/plugins/test_percentvolumechangepairlist.py diff --git a/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py new file mode 100644 index 000000000..0aee37959 --- /dev/null +++ b/freqtrade/plugins/pairlist/PercentVolumeChangePairList.py @@ -0,0 +1,316 @@ +""" +Change PairList provider + +Provides dynamic pair list based on trade change +sorted based on percentage change in volume over a +defined period +""" +import logging +from datetime import timedelta +from typing import Any, Dict, List, Literal + +from cachetools import TTLCache + +from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting +from freqtrade.util import dt_now, format_ms_time + + +logger = logging.getLogger(__name__) + +SORT_VALUES = ["rolling_volume_change"] + + +class PercentVolumeChangePairList(IPairList): + is_pairlist_generator = True + supports_backtesting = SupportsBacktesting.NO + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + if "number_assets" not in self._pairlistconfig: + raise OperationalException( + "`number_assets` not specified. Please check your configuration " + 'for "pairlist.config.number_assets"' + ) + + self._stake_currency = self._config["stake_currency"] + self._number_pairs = self._pairlistconfig["number_assets"] + self._sort_key: Literal["rolling_volume_change"] = self._pairlistconfig.get( + "sort_key", "rolling_volume_change" + ) + self._min_value = self._pairlistconfig.get("min_value", 0) + self._max_value = self._pairlistconfig.get("max_value", None) + self._refresh_period = self._pairlistconfig.get("refresh_period", 1800) + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._lookback_days = self._pairlistconfig.get("lookback_days", 0) + self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d") + self._lookback_period = self._pairlistconfig.get("lookback_period", 0) + self._def_candletype = self._config["candle_type_def"] + + if (self._lookback_days > 0) & (self._lookback_period > 0): + raise OperationalException( + "Ambiguous configuration: lookback_days and lookback_period both set in pairlist " + "config. Please set lookback_days only or lookback_period and lookback_timeframe " + "and restart the bot." + ) + + # overwrite lookback timeframe and days when lookback_days is set + if self._lookback_days > 0: + self._lookback_timeframe = "1d" + self._lookback_period = self._lookback_days + + # get timeframe in minutes and seconds + self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) + _tf_in_sec = self._tf_in_min * 60 + + # whether to use range lookback or not + self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) + + if self._use_range & (self._refresh_period < _tf_in_sec): + raise OperationalException( + f"Refresh period of {self._refresh_period} seconds is smaller than one " + f"timeframe of {self._lookback_timeframe}. Please adjust refresh_period " + f"to at least {_tf_in_sec} and restart the bot." + ) + + if not self._use_range and not ( + self._exchange.exchange_has("fetchTickers") + and self._exchange.get_option("tickers_have_change") + ): + raise OperationalException( + "Exchange does not support dynamic whitelist in this configuration. " + "Please edit your config and either remove PercentVolumeChangePairList, " + "or switch to using candles. and restart the bot." + ) + + if not self._validate_keys(self._sort_key): + raise OperationalException(f"key {self._sort_key} not in {SORT_VALUES}") + + candle_limit = self._exchange.ohlcv_candle_limit( + self._lookback_timeframe, self._config["candle_type_def"] + ) + if self._lookback_period < 4: + raise OperationalException("ChangeFilter requires lookback_period to be >= 4") + self.log_once(f"Candle limit is {candle_limit}", logger.info) + if self._lookback_period > candle_limit: + raise OperationalException( + "ChangeFilter requires lookback_period to not " + f"exceed exchange max request size ({candle_limit})" + ) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return not self._use_range + + def _validate_keys(self, key): + return key in SORT_VALUES + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - top {self._pairlistconfig['number_assets']} percent " + f"volume change pairs.") + + @staticmethod + def description() -> str: + return "Provides dynamic pair list based on percentage volume change." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 30, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "sort_key": { + "type": "option", + "default": "rolling_volume_change", + "options": SORT_VALUES, + "description": "Sort key", + "help": "Sort key to use for sorting the pairlist.", + }, + "min_value": { + "type": "number", + "default": 0, + "description": "Minimum value", + "help": "Minimum value to use for filtering the pairlist.", + }, + "max_value": { + "type": "number", + "default": None, + "description": "Maximum value", + "help": "Maximum value to use for filtering the pairlist.", + }, + "refresh_period": { + "type": "number", + "default": 1800, + "description": "Refresh period", + "help": "Refresh period in seconds", + }, + "lookback_days": { + "type": "number", + "default": 0, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "lookback_timeframe": { + "type": "string", + "default": "1d", + "description": "Lookback Timeframe", + "help": "Timeframe to use for lookback.", + }, + "lookback_period": { + "type": "number", + "default": 0, + "description": "Lookback Period", + "help": "Number of periods to look back at.", + }, + } + + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + pairlist = self._pair_cache.get("pairlist") + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + _pairlist = [ + k + for k in self._exchange.get_markets( + quote_currencies=[self._stake_currency], tradable_only=True, active_only=True + ).keys() + ] + + # No point in testing for blacklisted pairs... + _pairlist = self.verify_blacklist(_pairlist, logger.info) + if not self._use_range: + filtered_tickers = [ + v + for k, v in tickers.items() + if ( + self._exchange.get_pair_quote_currency(k) == self._stake_currency + and (self._use_range or v.get(self._sort_key) is not None) + and v["symbol"] in _pairlist + ) + ] + pairlist = [s["symbol"] for s in filtered_tickers] + else: + pairlist = _pairlist + + pairlist = self.filter_pairlist(pairlist, tickers) + self._pair_cache["pairlist"] = pairlist.copy() + + return pairlist + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning) + if self._use_range: + filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist] + + # get lookback period in ms, for exchange ohlcv fetch + since_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, + dt_now() + + timedelta( + minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min + ), + ).timestamp() + ) + * 1000 + ) + + to_ms = ( + int( + timeframe_to_prev_date( + self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min) + ).timestamp() + ) + * 1000 + ) + + # todo: utc date output for starting date + self.log_once( + f"Using change range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", + logger.info, + ) + needed_pairs: ListPairsWithTimeframes = [ + (p, self._lookback_timeframe, self._def_candletype) + for p in [s["symbol"] for s in filtered_tickers] + if p not in self._pair_cache + ] + + candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms) + + for i, p in enumerate(filtered_tickers): + pair_candles = ( + candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)] + if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles + else None + ) + + # in case of candle data calculate typical price and change for candle + if pair_candles is not None and not pair_candles.empty: + pair_candles["rolling_volume_sum"] = ( + pair_candles["volume"].rolling(window=self._lookback_period).sum() + ) + pair_candles["rolling_volume_change"] = ( + pair_candles["rolling_volume_sum"].pct_change() * 100 + ) + + # ensure that a rolling sum over the lookback_period is built + # if pair_candles contains more candles than lookback_period + rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1] + + # replace change with a range change sum calculated above + filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change + self.log_once(f"ticker {filtered_tickers[i]}", logger.info) + else: + filtered_tickers[i]["rolling_volume_change"] = 0 + else: + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + + filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value] + if self._max_value is not None: + filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value] + + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) + + self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info) + # Validate whitelist to only have active market pairs + pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers]) + pairs = self.verify_blacklist(pairs, logmethod=logger.info) + # Limit pairlist to the requested number of pairs + pairs = pairs[: self._number_pairs] + + return pairs diff --git a/tests/plugins/test_percentvolumechangepairlist.py b/tests/plugins/test_percentvolumechangepairlist.py new file mode 100644 index 000000000..b09307151 --- /dev/null +++ b/tests/plugins/test_percentvolumechangepairlist.py @@ -0,0 +1,376 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pandas as pd +import pytest + +from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.enums import CandleType +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList +from freqtrade.plugins.pairlistmanager import PairListManager +from tests.conftest import ( + EXMS, + generate_test_data_raw, + get_patched_exchange, + get_patched_freqtradebot, +) + + +@pytest.fixture(scope="function") +def rpl_config(default_conf): + default_conf["stake_currency"] = "USDT" + + default_conf["exchange"]["pair_whitelist"] = [ + "ETH/USDT", + "XRP/USDT", + ] + default_conf["exchange"]["pair_blacklist"] = ["BLK/USDT"] + + return default_conf + + +def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + } + ] + + with pytest.raises( + OperationalException, + match=r"Exchange does not support dynamic whitelist in this configuration. " + r"Please edit your config and either remove PercentVolumeChangePairList, " + r"or switch to using candles. and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 1800, + "lookback_days": 4, + } + ] + + with pytest.raises( + OperationalException, + match=r"Refresh period of 1800 seconds is smaller than one " + r"timeframe of 1d. Please adjust refresh_period " + r"to at least 86400 and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 3, + "lookback_period": 3, + } + ] + + with pytest.raises( + OperationalException, + match=r"Ambiguous configuration: lookback_days " + r"and lookback_period both set in pairlist config. " + r"Please set lookback_days only or lookback_period " + r"and lookback_timeframe and restart the bot.", + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 3, + } + ] + + with pytest.raises( + OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_period": 3, + } + ] + + with pytest.raises( + OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4" + ): + get_patched_freqtradebot(mocker, rpl_config) + + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1001, + } + ] + + with pytest.raises( + OperationalException, + match=r"ChangeFilter requires lookback_period to not exceed" + r" exchange max request size \(1000\)", + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + } + ] + + with pytest.raises( + OperationalException, + match=r"`number_assets` not specified. Please check your configuration " + r'for "pairlist.config.number_assets"', + ): + get_patched_freqtradebot(mocker, rpl_config) + + +def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=12), + "1d", + pair="ETH/USDT", + fill_missing=True, + ) + ), + ("BTC/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=13), + "1d", + pair="BTC/USDT", + fill_missing=True, + ) + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=14), + "1d", + pair="XRP/USDT", + fill_missing=True, + ) + ), + ("NEO/USDT", "1d", CandleType.SPOT): pd.DataFrame( + ohlcv_to_dataframe( + generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=15), + "1d", + pair="NEO/USDT", + fill_missing=True, + ) + ), + ("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame( + # Make sure always have highest rolling_volume_change + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.gen_pairlist(tickers) + + assert len(result) == 2 + assert result == ["TKN/USDT", "BTC/USDT"] + + +def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) + + assert len(result) == 2 + assert result == ["ETH/USDT", "XRP/USDT"] + + +def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine): + rpl_config["pairlists"] = [ + { + "method": "PercentVolumeChangePairList", + "number_assets": 2, + "sort_key": "rolling_volume_change", + "min_value": 0, + "max_value": 15, + "refresh_period": 86400, + "lookback_days": 4, + } + ] + + start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc) + time_machine.move_to(start, tick=False) + + mock_ohlcv_data = { + ("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 1800, 2400, 2500], + } + ), + ("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame( + { + "timestamp": [ + "2024-07-01 00:00:00", + "2024-07-01 01:00:00", + "2024-07-01 02:00:00", + "2024-07-01 03:00:00", + "2024-07-01 04:00:00", + "2024-07-01 05:00:00", + ], + "open": [100, 102, 101, 103, 104, 105], + "high": [102, 103, 102, 104, 105, 106], + "low": [99, 101, 100, 102, 103, 104], + "close": [101, 102, 103, 104, 105, 106], + "volume": [1000, 1500, 2000, 2500, 3000, 3500], + } + ), + } + + mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data)) + exchange = get_patched_exchange(mocker, rpl_config, exchange="binance") + pairlistmanager = PairListManager(exchange, rpl_config) + + remote_pairlist = PercentVolumeChangePairList( + exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0 + ) + + result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {}) + + assert len(result) == 1 + assert result == ["ETH/USDT"]