From 7c15375f5da6fb66efc00f356486eb4a5736534d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:20:36 +0200 Subject: [PATCH 1/6] Add log_on_refresh - using TTL caching to avoid spamming logs --- freqtrade/pairlist/IPairList.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 35844a99e..e57bc4e88 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -9,6 +9,8 @@ from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, Dict, List +from cachetools import TTLCache, cached + from freqtrade.exchange import market_is_active logger = logging.getLogger(__name__) @@ -31,6 +33,9 @@ class IPairList(ABC): self._config = config self._pairlistconfig = pairlistconfig self._pairlist_pos = pairlist_pos + self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._last_refresh = 0 + self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period) @property def name(self) -> str: @@ -40,6 +45,24 @@ class IPairList(ABC): """ return self.__class__.__name__ + def log_on_refresh(self, logmethod, message: str) -> None: + """ + Logs message - not more often than "refresh_period" to avoid log spamming + Logs the log-message as debug as well to simplify debugging. + :param logmethod: Function that'll be called. Most likely `logger.info`. + :param message: String containing the message to be sent to the function. + :return: None. + """ + + @cached(cache=self._log_cache) + def _log_on_refresh(logmethod, message: str): + logmethod(message) + + # Log as debug first + logger.debug(message) + # Call hidden function. + _log_on_refresh(logmethod, message) + @abstractproperty def needstickers(self) -> bool: """ From 5d876ca0a359f129a79a3f486216261734fd9003 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:21:10 +0200 Subject: [PATCH 2/6] Use log-spamprevention methods --- freqtrade/pairlist/PriceFilter.py | 4 ++-- freqtrade/pairlist/SpreadFilter.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index dc02ae251..7ea1e6003 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -43,8 +43,8 @@ class PriceFilter(IPairList): compare = ticker['last'] + 1 / pow(10, precision) changeperc = (compare - ticker['last']) / ticker['last'] if changeperc > self._low_price_ratio: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") + self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") return False return True diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py index 9361837cc..49731ef11 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/pairlist/SpreadFilter.py @@ -49,9 +49,9 @@ class SpreadFilter(IPairList): if 'bid' in ticker and 'ask' in ticker: spread = 1 - ticker['bid'] / ticker['ask'] if not ticker or spread > self._max_spread_ratio: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because spread {spread * 100:.3f}% >" - f"{self._max_spread_ratio * 100}%") + self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because spread {spread * 100:.3f}% >" + f"{self._max_spread_ratio * 100}%") pairlist.remove(p) else: pairlist.remove(p) From 13ee7a55c4756035d647f00a25767407e2cafe40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:21:30 +0200 Subject: [PATCH 3/6] Fix #3166 Always call _gen_pair_whitelist if volumepairlist is not the first in the list. --- freqtrade/pairlist/VolumePairList.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 9ce2adc9e..65f43245c 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -39,7 +39,6 @@ class VolumePairList(IPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - self._last_refresh = 0 @property def needstickers(self) -> bool: @@ -68,16 +67,18 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - if self._last_refresh + self.refresh_period < datetime.now().timestamp(): + # Must always run if this pairlist is not the first in the list. + if (self._pairlist_pos != 0 or + (self._last_refresh + self.refresh_period < datetime.now().timestamp())): + self._last_refresh = int(datetime.now().timestamp()) - return self._gen_pair_whitelist(pairlist, - tickers, - self._config['stake_currency'], - self._sort_key, - self._min_value - ) + pairs = self._gen_pair_whitelist(pairlist, tickers, + self._config['stake_currency'], + self._sort_key, self._min_value) else: - return pairlist + pairs = pairlist + self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + return pairs def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict, base_currency: str, key: str, min_val: int) -> List[str]: @@ -88,7 +89,6 @@ class VolumePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist # Check if pair quote currency equals to the stake currency. @@ -109,6 +109,5 @@ class VolumePairList(IPairList): pairs = self._verify_blacklist(pairs, aswarning=False) # Limit to X number of pairs pairs = pairs[:self._number_pairs] - logger.info(f"Searching {self._number_pairs} pairs: {pairs}") return pairs From ceca0a659cef27ed872bcb504af6ec74c99f1511 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:25:58 +0200 Subject: [PATCH 4/6] Simplify cached stuff to only what's needed --- freqtrade/pairlist/IPairList.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index e57bc4e88..e089e546c 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -55,13 +55,13 @@ class IPairList(ABC): """ @cached(cache=self._log_cache) - def _log_on_refresh(logmethod, message: str): + def _log_on_refresh(message: str): logmethod(message) # Log as debug first logger.debug(message) # Call hidden function. - _log_on_refresh(logmethod, message) + _log_on_refresh(message) @abstractproperty def needstickers(self) -> bool: From 1b2bf2c9b69680d2ed09e3edfeffc1d00451932a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:39:54 +0200 Subject: [PATCH 5/6] Add test for cached log method --- tests/pairlist/test_pairlist.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 1ce1151b7..6275bdafc 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -46,6 +46,28 @@ def static_pl_conf(whitelist_conf): return whitelist_conf +def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + logmock = MagicMock() + # Assign starting whitelist + pl = freqtrade.pairlists._pairlists[0] + pl.log_on_refresh(logmock, 'Hello world') + assert logmock.call_count == 1 + pl.log_on_refresh(logmock, 'Hello world') + assert logmock.call_count == 1 + assert pl._log_cache.currsize == 1 + assert ('Hello world',) in pl._log_cache._Cache__data + + pl.log_on_refresh(logmock, 'Hello world2') + assert logmock.call_count == 2 + assert pl._log_cache.currsize == 2 + + def test_load_pairlist_noexist(mocker, markets, default_conf): bot = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) From 2b7376f6f334d78ede1d98fb1fee5ca3af4a9b34 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Apr 2020 20:45:30 +0200 Subject: [PATCH 6/6] Implement log-filtering for all pairlists --- freqtrade/pairlist/PrecisionFilter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index f16458ca5..2a2ba46b7 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -39,8 +39,9 @@ class PrecisionFilter(IPairList): stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_on_refresh(logger.info, + f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") return False return True