CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-01-20 05:50:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

10348 | Create new pair list to dynamically fetch pairs based on percent volume change

Browse Source
This commit is contained in:
jainanuj94
2024-07-24 19:09:45 +05:30
parent e2def42932
commit 4b1177e07e
2 changed files with 692 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
freqtrade/plugins/pairlist/PercentVolumeChangePairList.py Normal file
View File

@@ -0,0 +1,316 @@
"""
Change PairList provider
Provides dynamic pair list based on trade change
sorted based on percentage change in volume over a
defined period
"""
import logging
from datetime import timedelta
from typing import Any, Dict, List, Literal
from cachetools import TTLCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
from freqtrade.util import dt_now, format_ms_time
logger = logging.getLogger(__name__)
SORT_VALUES = ["rolling_volume_change"]
class PercentVolumeChangePairList(IPairList):
is_pairlist_generator = True
supports_backtesting = SupportsBacktesting.NO
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if "number_assets" not in self._pairlistconfig:
raise OperationalException(
"`number_assets` not specified. Please check your configuration "
'for "pairlist.config.number_assets"'
)
self._stake_currency = self._config["stake_currency"]
self._number_pairs = self._pairlistconfig["number_assets"]
self._sort_key: Literal["rolling_volume_change"] = self._pairlistconfig.get(
"sort_key", "rolling_volume_change"
)
self._min_value = self._pairlistconfig.get("min_value", 0)
self._max_value = self._pairlistconfig.get("max_value", None)
self._refresh_period = self._pairlistconfig.get("refresh_period", 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._lookback_days = self._pairlistconfig.get("lookback_days", 0)
self._lookback_timeframe = self._pairlistconfig.get("lookback_timeframe", "1d")
self._lookback_period = self._pairlistconfig.get("lookback_period", 0)
self._def_candletype = self._config["candle_type_def"]
if (self._lookback_days > 0) & (self._lookback_period > 0):
raise OperationalException(
"Ambiguous configuration: lookback_days and lookback_period both set in pairlist "
"config. Please set lookback_days only or lookback_period and lookback_timeframe "
"and restart the bot."
)
# overwrite lookback timeframe and days when lookback_days is set
if self._lookback_days > 0:
self._lookback_timeframe = "1d"
self._lookback_period = self._lookback_days
# get timeframe in minutes and seconds
self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
_tf_in_sec = self._tf_in_min * 60
# whether to use range lookback or not
self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
if self._use_range & (self._refresh_period < _tf_in_sec):
raise OperationalException(
f"Refresh period of {self._refresh_period} seconds is smaller than one "
f"timeframe of {self._lookback_timeframe}. Please adjust refresh_period "
f"to at least {_tf_in_sec} and restart the bot."
)
if not self._use_range and not (
self._exchange.exchange_has("fetchTickers")
and self._exchange.get_option("tickers_have_change")
):
raise OperationalException(
"Exchange does not support dynamic whitelist in this configuration. "
"Please edit your config and either remove PercentVolumeChangePairList, "
"or switch to using candles. and restart the bot."
)
if not self._validate_keys(self._sort_key):
raise OperationalException(f"key {self._sort_key} not in {SORT_VALUES}")
candle_limit = self._exchange.ohlcv_candle_limit(
self._lookback_timeframe, self._config["candle_type_def"]
)
if self._lookback_period < 4:
raise OperationalException("ChangeFilter requires lookback_period to be >= 4")
self.log_once(f"Candle limit is {candle_limit}", logger.info)
if self._lookback_period > candle_limit:
raise OperationalException(
"ChangeFilter requires lookback_period to not "
f"exceed exchange max request size ({candle_limit})"
)
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return not self._use_range
def _validate_keys(self, key):
return key in SORT_VALUES
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - top {self._pairlistconfig['number_assets']} percent "
f"volume change pairs.")
@staticmethod
def description() -> str:
return "Provides dynamic pair list based on percentage volume change."
@staticmethod
def available_parameters() -> Dict[str, PairlistParameter]:
return {
"number_assets": {
"type": "number",
"default": 30,
"description": "Number of assets",
"help": "Number of assets to use from the pairlist",
},
"sort_key": {
"type": "option",
"default": "rolling_volume_change",
"options": SORT_VALUES,
"description": "Sort key",
"help": "Sort key to use for sorting the pairlist.",
},
"min_value": {
"type": "number",
"default": 0,
"description": "Minimum value",
"help": "Minimum value to use for filtering the pairlist.",
},
"max_value": {
"type": "number",
"default": None,
"description": "Maximum value",
"help": "Maximum value to use for filtering the pairlist.",
},
"refresh_period": {
"type": "number",
"default": 1800,
"description": "Refresh period",
"help": "Refresh period in seconds",
},
"lookback_days": {
"type": "number",
"default": 0,
"description": "Lookback Days",
"help": "Number of days to look back at.",
},
"lookback_timeframe": {
"type": "string",
"default": "1d",
"description": "Lookback Timeframe",
"help": "Timeframe to use for lookback.",
},
"lookback_period": {
"type": "number",
"default": 0,
"description": "Lookback Period",
"help": "Number of periods to look back at.",
},
}
def gen_pairlist(self, tickers: Tickers) -> List[str]:
"""
Generate the pairlist
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: List of pairs
"""
# Generate dynamic whitelist
# Must always run if this pairlist is not the first in the list.
pairlist = self._pair_cache.get("pairlist")
if pairlist:
# Item found - no refresh necessary
return pairlist.copy()
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
_pairlist = [
k
for k in self._exchange.get_markets(
quote_currencies=[self._stake_currency], tradable_only=True, active_only=True
).keys()
]
# No point in testing for blacklisted pairs...
_pairlist = self.verify_blacklist(_pairlist, logger.info)
if not self._use_range:
filtered_tickers = [
v
for k, v in tickers.items()
if (
self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v.get(self._sort_key) is not None)
and v["symbol"] in _pairlist
)
]
pairlist = [s["symbol"] for s in filtered_tickers]
else:
pairlist = _pairlist
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache["pairlist"] = pairlist.copy()
return pairlist
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: new whitelist
"""
self.log_once(f"Filter ticker is self use range {pairlist}", logger.warning)
if self._use_range:
filtered_tickers: List[Dict[str, Any]] = [{"symbol": k} for k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
since_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe,
dt_now()
+ timedelta(
minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min
),
).timestamp()
)
* 1000
)
to_ms = (
int(
timeframe_to_prev_date(
self._lookback_timeframe, dt_now() - timedelta(minutes=self._tf_in_min)
).timestamp()
)
* 1000
)
# todo: utc date output for starting date
self.log_once(
f"Using change range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}",
logger.info,
)
needed_pairs: ListPairsWithTimeframes = [
(p, self._lookback_timeframe, self._def_candletype)
for p in [s["symbol"] for s in filtered_tickers]
if p not in self._pair_cache
]
candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)
for i, p in enumerate(filtered_tickers):
pair_candles = (
candles[(p["symbol"], self._lookback_timeframe, self._def_candletype)]
if (p["symbol"], self._lookback_timeframe, self._def_candletype) in candles
else None
)
# in case of candle data calculate typical price and change for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles["rolling_volume_sum"] = (
pair_candles["volume"].rolling(window=self._lookback_period).sum()
)
pair_candles["rolling_volume_change"] = (
pair_candles["rolling_volume_sum"].pct_change() * 100
)
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
rolling_volume_change = pair_candles["rolling_volume_change"].fillna(0).iloc[-1]
# replace change with a range change sum calculated above
filtered_tickers[i]["rolling_volume_change"] = rolling_volume_change
self.log_once(f"ticker {filtered_tickers[i]}", logger.info)
else:
filtered_tickers[i]["rolling_volume_change"] = 0
else:
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] > self._min_value]
if self._max_value is not None:
filtered_tickers = [v for v in filtered_tickers if v[self._sort_key] < self._max_value]
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
self.log_once(f"Sorted Tickers {sorted_tickers}", logger.info)
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s["symbol"] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, logmethod=logger.info)
# Limit pairlist to the requested number of pairs
pairs = pairs[: self._number_pairs]
return pairs

376
tests/plugins/test_percentvolumechangepairlist.py Normal file
View File

@@ -0,0 +1,376 @@
from datetime import datetime, timezone
from unittest.mock import MagicMock
import pandas as pd
import pytest
from freqtrade.data.converter import ohlcv_to_dataframe
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.PercentVolumeChangePairList import PercentVolumeChangePairList
from freqtrade.plugins.pairlistmanager import PairListManager
from tests.conftest import (
EXMS,
generate_test_data_raw,
get_patched_exchange,
get_patched_freqtradebot,
)
@pytest.fixture(scope="function")
def rpl_config(default_conf):
default_conf["stake_currency"] = "USDT"
default_conf["exchange"]["pair_whitelist"] = [
"ETH/USDT",
"XRP/USDT",
]
default_conf["exchange"]["pair_blacklist"] = ["BLK/USDT"]
return default_conf
def test_volume_change_pair_list_init_exchange_support(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
}
]
with pytest.raises(
OperationalException,
match=r"Exchange does not support dynamic whitelist in this configuration. "
r"Please edit your config and either remove PercentVolumeChangePairList, "
r"or switch to using candles. and restart the bot.",
):
get_patched_freqtradebot(mocker, rpl_config)
def test_volume_change_pair_list_init_wrong_refresh_period(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 1800,
"lookback_days": 4,
}
]
with pytest.raises(
OperationalException,
match=r"Refresh period of 1800 seconds is smaller than one "
r"timeframe of 1d. Please adjust refresh_period "
r"to at least 86400 and restart the bot.",
):
get_patched_freqtradebot(mocker, rpl_config)
def test_volume_change_pair_list_init_wrong_lookback_period(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 3,
"lookback_period": 3,
}
]
with pytest.raises(
OperationalException,
match=r"Ambiguous configuration: lookback_days "
r"and lookback_period both set in pairlist config. "
r"Please set lookback_days only or lookback_period "
r"and lookback_timeframe and restart the bot.",
):
get_patched_freqtradebot(mocker, rpl_config)
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 3,
}
]
with pytest.raises(
OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4"
):
get_patched_freqtradebot(mocker, rpl_config)
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_period": 3,
}
]
with pytest.raises(
OperationalException, match=r"ChangeFilter requires lookback_period to be >= 4"
):
get_patched_freqtradebot(mocker, rpl_config)
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 1001,
}
]
with pytest.raises(
OperationalException,
match=r"ChangeFilter requires lookback_period to not exceed"
r" exchange max request size \(1000\)",
):
get_patched_freqtradebot(mocker, rpl_config)
def test_volume_change_pair_list_init_wrong_config(mocker, rpl_config):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
}
]
with pytest.raises(
OperationalException,
match=r"`number_assets` not specified. Please check your configuration "
r'for "pairlist.config.number_assets"',
):
get_patched_freqtradebot(mocker, rpl_config)
def test_gen_pairlist_with_valid_change_pair_list_config(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 4,
}
]
start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
time_machine.move_to(start, tick=False)
mock_ohlcv_data = {
("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame(
ohlcv_to_dataframe(
generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=12),
"1d",
pair="ETH/USDT",
fill_missing=True,
)
),
("BTC/USDT", "1d", CandleType.SPOT): pd.DataFrame(
ohlcv_to_dataframe(
generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=13),
"1d",
pair="BTC/USDT",
fill_missing=True,
)
),
("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame(
ohlcv_to_dataframe(
generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=14),
"1d",
pair="XRP/USDT",
fill_missing=True,
)
),
("NEO/USDT", "1d", CandleType.SPOT): pd.DataFrame(
ohlcv_to_dataframe(
generate_test_data_raw("1d", 100, start.strftime("%Y-%m-%d"), random_seed=15),
"1d",
pair="NEO/USDT",
fill_missing=True,
)
),
("TKN/USDT", "1d", CandleType.SPOT): pd.DataFrame(
# Make sure always have highest rolling_volume_change
{
"timestamp": [
"2024-07-01 00:00:00",
"2024-07-01 01:00:00",
"2024-07-01 02:00:00",
"2024-07-01 03:00:00",
"2024-07-01 04:00:00",
"2024-07-01 05:00:00",
],
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
}
mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data))
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.gen_pairlist(tickers)
assert len(result) == 2
assert result == ["TKN/USDT", "BTC/USDT"]
def test_filter_pairlist_with_empty_ticker(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"refresh_period": 86400,
"lookback_days": 4,
}
]
start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
time_machine.move_to(start, tick=False)
mock_ohlcv_data = {
("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame(
{
"timestamp": [
"2024-07-01 00:00:00",
"2024-07-01 01:00:00",
"2024-07-01 02:00:00",
"2024-07-01 03:00:00",
"2024-07-01 04:00:00",
"2024-07-01 05:00:00",
],
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame(
{
"timestamp": [
"2024-07-01 00:00:00",
"2024-07-01 01:00:00",
"2024-07-01 02:00:00",
"2024-07-01 03:00:00",
"2024-07-01 04:00:00",
"2024-07-01 05:00:00",
],
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
}
mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data))
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {})
assert len(result) == 2
assert result == ["ETH/USDT", "XRP/USDT"]
def test_filter_pairlist_with_max_value_set(mocker, rpl_config, tickers, time_machine):
rpl_config["pairlists"] = [
{
"method": "PercentVolumeChangePairList",
"number_assets": 2,
"sort_key": "rolling_volume_change",
"min_value": 0,
"max_value": 15,
"refresh_period": 86400,
"lookback_days": 4,
}
]
start = datetime(2024, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
time_machine.move_to(start, tick=False)
mock_ohlcv_data = {
("ETH/USDT", "1d", CandleType.SPOT): pd.DataFrame(
{
"timestamp": [
"2024-07-01 00:00:00",
"2024-07-01 01:00:00",
"2024-07-01 02:00:00",
"2024-07-01 03:00:00",
"2024-07-01 04:00:00",
"2024-07-01 05:00:00",
],
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 1800, 2400, 2500],
}
),
("XRP/USDT", "1d", CandleType.SPOT): pd.DataFrame(
{
"timestamp": [
"2024-07-01 00:00:00",
"2024-07-01 01:00:00",
"2024-07-01 02:00:00",
"2024-07-01 03:00:00",
"2024-07-01 04:00:00",
"2024-07-01 05:00:00",
],
"open": [100, 102, 101, 103, 104, 105],
"high": [102, 103, 102, 104, 105, 106],
"low": [99, 101, 100, 102, 103, 104],
"close": [101, 102, 103, 104, 105, 106],
"volume": [1000, 1500, 2000, 2500, 3000, 3500],
}
),
}
mocker.patch(f"{EXMS}.refresh_latest_ohlcv", MagicMock(return_value=mock_ohlcv_data))
exchange = get_patched_exchange(mocker, rpl_config, exchange="binance")
pairlistmanager = PairListManager(exchange, rpl_config)
remote_pairlist = PercentVolumeChangePairList(
exchange, pairlistmanager, rpl_config, rpl_config["pairlists"][0], 0
)
result = remote_pairlist.filter_pairlist(rpl_config["exchange"]["pair_whitelist"], {})
assert len(result) == 1
assert result == ["ETH/USDT"]