From 457e738b4a049cbc522987612dc2e4bd91b272e9 Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Thu, 16 Sep 2021 14:48:02 +0300 Subject: [PATCH 1/7] Added days parameter to PerformanceFilter --- freqtrade/persistence/models.py | 28 ++++++++++++++++++- .../plugins/pairlist/PerformanceFilter.py | 15 ++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 8c8c1e0a9..91a26eba7 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,7 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from decimal import Decimal from typing import Any, Dict, List, Optional @@ -856,6 +856,32 @@ class Trade(_DECL_BASE, LocalTrade): for pair, profit, profit_abs, count in pair_rates ] + @staticmethod + def get_performance(days: int) -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, including profit and trade count + NOTE: Not supported in Backtesting. + """ + start_date = datetime.today() - timedelta(days) + pair_rates = Trade.query.with_entities( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum_abs')) \ + .all() + return [ + { + 'pair': pair, + 'profit': profit, + 'profit_abs': profit_abs, + 'count': count + } + for pair, profit, profit_abs, count in pair_rates + ] + @staticmethod def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)): """ diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 46a289ae6..4d530fe88 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,11 +2,12 @@ Performance pair list filter """ import logging -from typing import Dict, List +from typing import Dict, List, Any import pandas as pd from freqtrade.persistence import Trade +from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -15,6 +16,13 @@ logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('days', 0) + @property def needstickers(self) -> bool: """ @@ -40,7 +48,10 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - performance = pd.DataFrame(Trade.get_overall_performance()) + if self._days > 0: + performance = pd.DataFrame(Trade.get_performance(self._days)) + else: + performance = pd.DataFrame(Trade.get_overall_performance()) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 54ef36a4971d77467a10fa343857aec27a312e5a Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Fri, 17 Sep 2021 13:45:44 +0300 Subject: [PATCH 2/7] Updates after review to PerformanceFilter days param --- freqtrade/persistence/models.py | 34 ++++--------------- .../plugins/pairlist/PerformanceFilter.py | 5 +-- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 91a26eba7..222be7169 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -832,43 +832,21 @@ class Trade(_DECL_BASE, LocalTrade): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance() -> List[Dict[str, Any]]: + def get_overall_performance(days=None) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ + filters = [Trade.is_open.is_(False)] + if days: + start_date = datetime.today() - timedelta(days) + filters.append((Trade.close_date >= start_date)) pair_rates = Trade.query.with_entities( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False))\ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum_abs')) \ - .all() - return [ - { - 'pair': pair, - 'profit': profit, - 'profit_abs': profit_abs, - 'count': count - } - for pair, profit, profit_abs, count in pair_rates - ] - - @staticmethod - def get_performance(days: int) -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, including profit and trade count - NOTE: Not supported in Backtesting. - """ - start_date = datetime.today() - timedelta(days) - pair_rates = Trade.query.with_entities( - Trade.pair, - func.sum(Trade.close_profit).label('profit_sum'), - func.sum(Trade.close_profit_abs).label('profit_sum_abs'), - func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date))\ + ).filter(filters)\ .group_by(Trade.pair) \ .order_by(desc('profit_sum_abs')) \ .all() diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 4d530fe88..920b884a0 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -48,10 +48,7 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - if self._days > 0: - performance = pd.DataFrame(Trade.get_performance(self._days)) - else: - performance = pd.DataFrame(Trade.get_overall_performance()) + performance = pd.DataFrame(Trade.get_overall_performance(self._days)) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 982deeedf04cb1be9c5ae7b1ec3598a9d09a4b9a Mon Sep 17 00:00:00 2001 From: sergeykhliustin <51409210+sergeykhliustin@users.noreply.github.com> Date: Fri, 17 Sep 2021 18:23:13 +0300 Subject: [PATCH 3/7] Update freqtrade/persistence/models.py Co-authored-by: Matthias --- freqtrade/persistence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 222be7169..a2e30d58c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -846,7 +846,7 @@ class Trade(_DECL_BASE, LocalTrade): func.sum(Trade.close_profit).label('profit_sum'), func.sum(Trade.close_profit_abs).label('profit_sum_abs'), func.count(Trade.pair).label('count') - ).filter(filters)\ + ).filter(*filters)\ .group_by(Trade.pair) \ .order_by(desc('profit_sum_abs')) \ .all() From d7395e873be5560932752e00200c70fc2cd169c3 Mon Sep 17 00:00:00 2001 From: Sergey Khliustin Date: Fri, 17 Sep 2021 22:05:57 +0300 Subject: [PATCH 4/7] Removed unused OperationalException --- freqtrade/plugins/pairlist/PerformanceFilter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 920b884a0..5750b721c 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -7,7 +7,6 @@ from typing import Dict, List, Any import pandas as pd from freqtrade.persistence import Trade -from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList From 12c12d42df5e0f838aad1ef971ca8530345d2d00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 08:26:48 +0200 Subject: [PATCH 5/7] Add documentation for days parameter in PerformanceFilter --- docs/includes/pairlists.md | 14 ++++++++++++++ freqtrade/persistence/models.py | 4 ++-- freqtrade/plugins/pairlist/PerformanceFilter.py | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 69e12d5dc..71165f93b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -165,6 +165,7 @@ Example to remove the first 10 pairs from the pairlist: ```json "pairlists": [ + // ... { "method": "OffsetFilter", "offset": 10 @@ -190,6 +191,19 @@ Sorts pairs by past trade performance, as follows: Trade count is used as a tie breaker. +You can use the `days` parameter to only consider performance of the past X days. +Not defining this parameter (or setting it to 0) will use all-time performance. + +```json +"pairlists": [ + // ... + { + "method": "PerformanceFilter", + "days": 10 + } +], +``` + !!! Note `PerformanceFilter` does not support backtesting mode. diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index a2e30d58c..f5f4e3a26 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,7 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from decimal import Decimal from typing import Any, Dict, List, Optional @@ -840,7 +840,7 @@ class Trade(_DECL_BASE, LocalTrade): filters = [Trade.is_open.is_(False)] if days: start_date = datetime.today() - timedelta(days) - filters.append((Trade.close_date >= start_date)) + filters.append(Trade.close_date >= start_date) pair_rates = Trade.query.with_entities( Trade.pair, func.sum(Trade.close_profit).label('profit_sum'), diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 5750b721c..ee443b0fe 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,7 +2,7 @@ Performance pair list filter """ import logging -from typing import Dict, List, Any +from typing import Any, Dict, List import pandas as pd From 564e0b9a1ac89c19f2a9c1e94c04cafd4fd96afe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 08:36:06 +0200 Subject: [PATCH 6/7] Switch performanceFilter to use Minutes lookback resolution closes #5060 --- docs/includes/pairlists.md | 4 ++-- freqtrade/persistence/models.py | 6 +++--- freqtrade/plugins/pairlist/PerformanceFilter.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 71165f93b..b612a4ddf 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -191,7 +191,7 @@ Sorts pairs by past trade performance, as follows: Trade count is used as a tie breaker. -You can use the `days` parameter to only consider performance of the past X days. +You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window). Not defining this parameter (or setting it to 0) will use all-time performance. ```json @@ -199,7 +199,7 @@ Not defining this parameter (or setting it to 0) will use all-time performance. // ... { "method": "PerformanceFilter", - "days": 10 + "minutes": 1440 // rolling 24h } ], ``` diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f5f4e3a26..bc5ef961a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -832,14 +832,14 @@ class Trade(_DECL_BASE, LocalTrade): return total_open_stake_amount or 0 @staticmethod - def get_overall_performance(days=None) -> List[Dict[str, Any]]: + def get_overall_performance(minutes=None) -> List[Dict[str, Any]]: """ Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ filters = [Trade.is_open.is_(False)] - if days: - start_date = datetime.today() - timedelta(days) + if minutes: + start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) filters.append(Trade.close_date >= start_date) pair_rates = Trade.query.with_entities( Trade.pair, diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index ee443b0fe..301ee57ab 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -20,7 +20,7 @@ class PerformanceFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._days = pairlistconfig.get('days', 0) + self._minutes = pairlistconfig.get('minutes', 0) @property def needstickers(self) -> bool: @@ -47,7 +47,7 @@ class PerformanceFilter(IPairList): """ # Get the trading performance for pairs from database try: - performance = pd.DataFrame(Trade.get_overall_performance(self._days)) + performance = pd.DataFrame(Trade.get_overall_performance(self._minutes)) except AttributeError: # Performancefilter does not work in backtesting. self.log_once("PerformanceFilter is not available in this mode.", logger.warning) From 56fb25c5e521415c9880e47c4938bfe357550bec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Sep 2021 09:10:37 +0200 Subject: [PATCH 7/7] Add test for PerformanceFilter lookback --- tests/plugins/test_pairlist.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 34770c03d..1ce8d172c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -12,7 +12,8 @@ from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re +from tests.conftest import (create_mock_trades, get_patched_exchange, get_patched_freqtradebot, + log_has, log_has_re) @pytest.fixture(scope="function") @@ -663,6 +664,31 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +@pytest.mark.usefixtures("init_persistence") +def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None: + whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "PerformanceFilter", "minutes": 60} + ] + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, whitelist_conf) + pm = PairListManager(exchange, whitelist_conf) + pm.refresh_pairlist() + + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + create_mock_trades(fee) + pm.refresh_pairlist() + assert pm.whitelist == ['XRP/BTC', 'ETH/BTC', 'TKN/BTC'] + + # Move to "outside" of lookback window, so original sorting is restored. + t.move_to("2021-09-01 07:00:00 +00:00") + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/BTC', 'TKN/BTC', 'XRP/BTC'] + + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}]