From 878e16545d3532813e9551608ea879651429a96d Mon Sep 17 00:00:00 2001 From: cdimauro Date: Thu, 29 Jul 2021 07:34:21 +0200 Subject: [PATCH 1/6] Suppress additional logs for pairs in blacklist Every time that there's freqtrade "ticks", pairs in the blacklist are checked and a warning message is displayed. So, the logs are continuously flooded with the same warnings. For example: 2021-07-26 06:24:45 freqtrade.plugins.pairlistmanager: WARNING - Pair XTZUP/USDT in your blacklist. Removing it from whitelist... 2021-07-26 06:24:45 freqtrade.plugins.pairlistmanager: WARNING - Pair SUSHIUP/USDT in your blacklist. Removing it from whitelist... 2021-07-26 06:24:45 freqtrade.plugins.pairlistmanager: WARNING - Pair XTZDOWN/USDT in your blacklist. Removing it from whitelist... 2021-07-26 06:24:50 freqtrade.plugins.pairlistmanager: WARNING - Pair XTZUP/USDT in your blacklist. Removing it from whitelist... 2021-07-26 06:24:50 freqtrade.plugins.pairlistmanager: WARNING - Pair SUSHIUP/USDT in your blacklist. Removing it from whitelist... 2021-07-26 06:24:50 freqtrade.plugins.pairlistmanager: WARNING - Pair XTZDOWN/USDT in your blacklist. Removing it from whitelist... This patch shows the warning only the first time, by keeping track of which pairs in the blacklist were already logged. --- freqtrade/plugins/pairlistmanager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index face79729..76ac84000 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -24,6 +24,7 @@ class PairListManager(): self._config = config self._whitelist = self._config['exchange'].get('pair_whitelist') self._blacklist = self._config['exchange'].get('pair_blacklist', []) + self._logged_blacklist_pairs = set() self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): @@ -108,9 +109,11 @@ class PairListManager(): except ValueError as err: logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") return [] - for pair in deepcopy(pairlist): + for pair in pairlist.copy(): if pair in blacklist: - logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") + if pair not in self._logged_blacklist_pairs: + logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") + self._logged_blacklist_pairs.add(pair) pairlist.remove(pair) return pairlist From 5b998aeca7f6bddea1da05e8883f9297187faa78 Mon Sep 17 00:00:00 2001 From: cdimauro Date: Sun, 12 Dec 2021 10:21:54 +0100 Subject: [PATCH 2/6] Remove unused import Remove the import from copy, since deepcopy() isn't used anymore (list.copy() is used instead). --- freqtrade/plugins/pairlistmanager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 76ac84000..d16eb50b6 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -2,7 +2,6 @@ PairList manager class """ import logging -from copy import deepcopy from typing import Dict, List from cachetools import TTLCache, cached From 8dd3128ed4f4d9027cee0e232db808cf8680e5a4 Mon Sep 17 00:00:00 2001 From: cdimauro Date: Sun, 12 Dec 2021 12:32:09 +0100 Subject: [PATCH 3/6] Add type annotation to new logs suppression code --- freqtrade/plugins/pairlistmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index d16eb50b6..02c6b5172 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -2,7 +2,7 @@ PairList manager class """ import logging -from typing import Dict, List +from typing import Dict, List, Set from cachetools import TTLCache, cached @@ -23,7 +23,7 @@ class PairListManager(): self._config = config self._whitelist = self._config['exchange'].get('pair_whitelist') self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._logged_blacklist_pairs = set() + self._logged_blacklist_pairs: Set[str] = set() self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): From 9d8646072c4c5f0dbf27eccca256428d26bd8de0 Mon Sep 17 00:00:00 2001 From: cdimauro Date: Tue, 14 Dec 2021 06:23:40 +0100 Subject: [PATCH 4/6] Add test case for checking removal of logs for pains in blacklist --- tests/plugins/test_pairlist.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 0d0e43b0b..f8a47382f 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access +import logging import time from unittest.mock import MagicMock, PropertyMock @@ -17,6 +18,9 @@ from tests.conftest import (create_mock_trades, get_patched_exchange, get_patche log_has, log_has_re) +logger = logging.getLogger(__name__) + + @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' @@ -217,6 +221,36 @@ def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog) +def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_conf, caplog): + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + whitelist = ['ETH/BTC', 'TKN/BTC'] + # Ensure all except those in whitelist are removed. + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + # Ensure that log message wasn't generated. + assert not log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...', caplog) + + new_whitelist = freqtrade.pairlists.verify_blacklist(whitelist + ['BLK/BTC'], logger.warning) + # Ensure that the pair is removed from the white list, and properly logged. + assert set(whitelist) == set(new_whitelist) + matches = sum(1 for message in caplog.messages + if message == 'Pair BLK/BTC in your blacklist. Removing it from whitelist...') + assert matches == 1 + + new_whitelist = freqtrade.pairlists.verify_blacklist(whitelist + ['BLK/BTC'], logger.warning) + # Ensure that the pair is not logged anymore when being removed from the pair list. + assert set(whitelist) == set(new_whitelist) + matches = sum(1 for message in caplog.messages + if message == 'Pair BLK/BTC in your blacklist. Removing it from whitelist...') + assert matches == 1 + + def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): mocker.patch.multiple( From 6ba8b17fdd2aeb320b17bf234082482464dcc9d2 Mon Sep 17 00:00:00 2001 From: cdimauro Date: Tue, 21 Dec 2021 09:11:57 +0100 Subject: [PATCH 5/6] Use LoggingMixin.log_once to remove/reduce logs on pairlists --- freqtrade/plugins/pairlistmanager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 02c6b5172..5e1378f14 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -2,12 +2,14 @@ PairList manager class """ import logging -from typing import Dict, List, Set +from functools import partial +from typing import Dict, List from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException +from freqtrade.mixins import LoggingMixin from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import PairListResolver @@ -16,14 +18,13 @@ from freqtrade.resolvers import PairListResolver logger = logging.getLogger(__name__) -class PairListManager(): +class PairListManager(LoggingMixin): def __init__(self, exchange, config: dict) -> None: self._exchange = exchange self._config = config self._whitelist = self._config['exchange'].get('pair_whitelist') self._blacklist = self._config['exchange'].get('pair_blacklist', []) - self._logged_blacklist_pairs: Set[str] = set() self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): @@ -41,6 +42,9 @@ class PairListManager(): if not self._pairlist_handlers: raise OperationalException("No Pairlist Handlers defined") + refresh_period = config.get('refresh_period', 1800) + LoggingMixin.__init__(self, logger, refresh_period) + @property def whitelist(self) -> List[str]: """The current whitelist""" @@ -108,11 +112,10 @@ class PairListManager(): except ValueError as err: logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") return [] + log_once = partial(self.log_once, logmethod=logmethod) for pair in pairlist.copy(): if pair in blacklist: - if pair not in self._logged_blacklist_pairs: - logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") - self._logged_blacklist_pairs.add(pair) + log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist From e5aaef6440ab0d0d44c99623f5697ee4322860cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 21 Dec 2021 19:20:09 +0100 Subject: [PATCH 6/6] Fix CI failure --- freqtrade/plugins/pairlistmanager.py | 2 +- tests/plugins/test_pairlist.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 5e1378f14..1e79dc743 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -42,7 +42,7 @@ class PairListManager(LoggingMixin): if not self._pairlist_handlers: raise OperationalException("No Pairlist Handlers defined") - refresh_period = config.get('refresh_period', 1800) + refresh_period = config.get('pairlist_refresh_period', 3600) LoggingMixin.__init__(self, logger, refresh_period) @property diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f8a47382f..b0f86eb7a 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -18,9 +18,6 @@ from tests.conftest import (create_mock_trades, get_patched_exchange, get_patche log_has, log_has_re) -logger = logging.getLogger(__name__) - - @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' @@ -222,6 +219,7 @@ def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_conf, caplog): + logger = logging.getLogger(__name__) freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -230,6 +228,9 @@ def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_c ) freqtrade.pairlists.refresh_pairlist() whitelist = ['ETH/BTC', 'TKN/BTC'] + caplog.clear() + caplog.set_level(logging.INFO) + # Ensure all except those in whitelist are removed. assert set(whitelist) == set(freqtrade.pairlists.whitelist) assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist