diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3e8874bb6..d8f789e6a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -12,8 +12,6 @@ from typing import Any, Callable, Dict, List, Optional import arrow from requests.exceptions import RequestException -from cachetools import TTLCache, cached - from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange @@ -25,7 +23,8 @@ from freqtrade.resolvers import StrategyResolver from freqtrade.state import State from freqtrade.strategy.interface import SellType, IStrategy from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -from freqtrade.pairlist.StaticList import StaticList +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade.pairlist.VolumePairList import VolumePairList logger = logging.getLogger(__name__) @@ -60,7 +59,11 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - self.pairlists = StaticList(self, self.config) + if self.config.get('dynamic_whitelist', None): + self.pairlists = VolumePairList(self, self.config) + + else: + self.pairlists = StaticPairList(self, self.config) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -148,18 +151,9 @@ class FreqtradeBot(object): """ state_changed = False try: - nb_assets = self.config.get('dynamic_whitelist', None) - # Refresh whitelist based on wallet maintenance + # Refresh whitelist self.pairlists.refresh_whitelist() - sanitized_list = self.pairlists.whitelist - # sanitized_list = self._refresh_whitelist( - # self._gen_pair_whitelist( - # self.config['stake_currency'] - # ) if nb_assets else self.lists.get_whitelist() - # ) - - # Keep only the subsets of pairs wanted (up to nb_assets) - self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list + self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong # Should be called before refresh_tickers @@ -207,30 +201,6 @@ class FreqtradeBot(object): self.state = State.STOPPED return state_changed - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') - :return: List of pairs - """ - - if not self.exchange.exchange_has('fetchTickers'): - raise OperationalException( - 'Exchange does not support dynamic whitelist.' - 'Please edit your config and restart the bot' - ) - - tickers = self.exchange.get_tickers() - # check length so that we make sure that '/' is actually in the string - tickers = [v for k, v in tickers.items() - if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] - - sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) - pairs = [s['symbol'] for s in sorted_tickers] - return pairs - def get_target_bid(self, pair: str, ticker: Dict[str, float]) -> float: """ Calculates bid target between current ask price and last price diff --git a/freqtrade/pairlist/StaticList.py b/freqtrade/pairlist/StaticPairList.py similarity index 91% rename from freqtrade/pairlist/StaticList.py rename to freqtrade/pairlist/StaticPairList.py index 3c46fab11..8466d7b82 100644 --- a/freqtrade/pairlist/StaticList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -10,7 +10,7 @@ from typing import List, Optional logger = logging.getLogger(__name__) -class StaticList(object): +class StaticPairList(object): def __init__(self, freqtrade, config: dict) -> None: self._freqtrade = freqtrade @@ -32,9 +32,9 @@ class StaticList(object): """ Refreshes whitelist and assigns it to self._whitelist """ - self._whitelist = self.validate_whitelist(self._config['exchange']['pair_whitelist']) + self._whitelist = self._validate_whitelist(self._config['exchange']['pair_whitelist']) - def validate_whitelist(self, whitelist: List[str]) -> List[str]: + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py new file mode 100644 index 000000000..c79f3c5c0 --- /dev/null +++ b/freqtrade/pairlist/VolumePairList.py @@ -0,0 +1,98 @@ +""" +Static List provider + +Provides lists as configured in config.json + + """ +import logging +from typing import List +from cachetools import TTLCache, cached + +from freqtrade.pairlist.StaticPairList import StaticPairList +from freqtrade import OperationalException +logger = logging.getLogger(__name__) + + +class VolumePairList(StaticPairList): + + def __init__(self, freqtrade, config: dict) -> None: + self._freqtrade = freqtrade + self._config = config + self._whitelist = self._config['exchange']['pair_whitelist'] + self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._number_pairs = self._config.get('dynamic_whitelist', None) + if not self._freqtrade.exchange.exchange_has('fetchTickers'): + raise OperationalException( + 'Exchange does not support dynamic whitelist.' + 'Please edit your config and restart the bot' + ) + # self.refresh_whitelist() + + @property + def whitelist(self) -> List[str]: + """ Contains the current whitelist """ + return self._whitelist + + @property + def blacklist(self) -> List[str]: + return self._blacklist + + def refresh_whitelist(self) -> None: + """ + Refreshes whitelist and assigns it to self._whitelist + """ + # Generate dynamic whitelist + pairs = self._gen_pair_whitelist(self._config['stake_currency']) + # Validate whitelist to only have active market pairs + self._whitelist = self._validate_whitelist(pairs)[:self._number_pairs] + + @cached(TTLCache(maxsize=1, ttl=1800)) + def _gen_pair_whitelist(self, base_currency: str, key: str = 'quoteVolume') -> List[str]: + """ + Updates the whitelist with with a dynamically generated list + :param base_currency: base currency as str + :param key: sort key (defaults to 'quoteVolume') + :return: List of pairs + """ + + tickers = self._freqtrade.exchange.get_tickers() + # check length so that we make sure that '/' is actually in the string + tickers = [v for k, v in tickers.items() + if len(k.split('/')) == 2 and k.split('/')[1] == base_currency] + + sorted_tickers = sorted(tickers, reverse=True, key=lambda t: t[key]) + pairs = [s['symbol'] for s in sorted_tickers] + return pairs + + def _validate_whitelist(self, whitelist: List[str]) -> List[str]: + """ + Check available markets and remove pair from whitelist if necessary + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to + trade + :return: the list of pairs the user wants to trade without the one unavailable or + black_listed + """ + sanitized_whitelist = whitelist + markets = self._freqtrade.exchange.get_markets() + + # Filter to markets in stake currency + markets = [m for m in markets if m['quote'] == self._config['stake_currency']] + known_pairs = set() + + for market in markets: + pair = market['symbol'] + # pair is not int the generated dynamic market, or in the blacklist ... ignore it + if pair not in whitelist or pair in self.blacklist: + continue + # else the pair is valid + known_pairs.add(pair) + # Market is not active + if not market['active']: + sanitized_whitelist.remove(pair) + logger.info( + 'Ignoring %s from whitelist. Market is not active.', + pair + ) + + # We need to remove pairs that are unknown + return [x for x in sanitized_whitelist if x in known_pairs] diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index d9fd93d09..997c171ad 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock +from freqtrade import OperationalException from freqtrade.tests.conftest import get_patched_freqtradebot import pytest @@ -35,7 +36,7 @@ def test_refresh_market_pair_not_in_whitelist(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist) == set(freqtradebot.pairlists.whitelist) # Ensure config dict hasn't been changed assert (whitelist_conf['exchange']['pair_whitelist'] == freqtradebot.config['exchange']['pair_whitelist']) @@ -49,11 +50,12 @@ def test_refresh_pairlists(mocker, markets, whitelist_conf): # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert whitelist == freqtradebot.pairlists.whitelist + assert set(whitelist)== set(freqtradebot.pairlists.whitelist) assert whitelist_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): + whitelist_conf['dynamic_whitelist'] = 5 freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -64,10 +66,7 @@ def test_refresh_whitelist_dynamic(mocker, markets, tickers, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC'] - - freqtradebot._refresh_whitelist( - freqtradebot._gen_pair_whitelist(whitelist_conf['stake_currency']) - ) + freqtradebot.pairlists.refresh_whitelist() assert whitelist == freqtradebot.pairlists.whitelist @@ -79,7 +78,40 @@ def test_refresh_whitelist_dynamic_empty(mocker, markets_empty, whitelist_conf): # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot._refresh_whitelist(whitelist) + freqtradebot.pairlists.refresh_whitelist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) + + +def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + # Test to retrieved BTC sorted on quoteVolume (default) + freqtrade.pairlists.refresh_whitelist() + + whitelist = freqtrade.pairlists.whitelist + assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] + + # Test to retrieve BTC sorted on bidVolume + whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') + assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] + + # Test with USDT sorted on quoteVolume (default) + whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') + assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] + + # Test with ETH (our fixture does not have ETH, so result should be empty) + whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') + assert whitelist == [] + + +def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + + with pytest.raises(OperationalException): + freqtrade._gen_pair_whitelist(base_currency='BTC') diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index a9b3ffc5d..71d451b94 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -136,37 +136,6 @@ def test_throttle_with_assets(mocker, default_conf) -> None: assert result == -1 -def test_gen_pair_whitelist(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - # Test to retrieved BTC sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC') - assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC'] - - # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'BLK/BTC'] - - # Test with USDT sorted on quoteVolume (default) - whitelist = freqtrade._gen_pair_whitelist(base_currency='USDT') - assert whitelist == ['TKN/USDT', 'ETH/USDT', 'LTC/USDT', 'BLK/USDT'] - - # Test with ETH (our fixture does not have ETH, so result should be empty) - whitelist = freqtrade._gen_pair_whitelist(base_currency='ETH') - assert whitelist == [] - - -def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) - - with pytest.raises(OperationalException): - freqtrade._gen_pair_whitelist(base_currency='BTC') - - def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker)