From 7fcbe9788d507b45c98fbbe278b4070e4a561154 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Jan 2024 19:53:06 +0100 Subject: [PATCH 1/5] Extract database cleanup functions to persistence package --- freqtrade/optimize/backtesting.py | 10 ++++------ freqtrade/persistence/__init__.py | 1 + freqtrade/persistence/usedb_context.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 freqtrade/persistence/usedb_context.py diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ce37a0dcc..137e51de5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,7 +33,8 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera show_backtest_results, store_backtest_analysis_results, store_backtest_stats) -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use, + enable_database_use) from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -177,8 +178,7 @@ class Backtesting: @staticmethod def cleanup(): LoggingMixin.show_output = True - PairLocks.use_db = True - Trade.use_db = True + enable_database_use() def init_backtest_detail(self) -> None: # Load detail timeframe if specified @@ -325,9 +325,7 @@ class Backtesting: self.futures_data = {} def disable_database_use(self): - PairLocks.use_db = False - PairLocks.timeframe = self.timeframe - Trade.use_db = False + disable_database_use(self.timeframe) def prepare_backtest(self, enable_protections): """ diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 4cf7aa455..595090540 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -4,3 +4,4 @@ from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade +from freqtrade.persistence.usedb_context import disable_database_use, enable_database_use diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py new file mode 100644 index 000000000..4d8c9e539 --- /dev/null +++ b/freqtrade/persistence/usedb_context.py @@ -0,0 +1,22 @@ + +from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.persistence.trade_model import Trade + + +def disable_database_use(timeframe: str) -> None: + """ + Disable database usage for PairLocks and Trade models. + Used for backtesting, and some other utility commands. + """ + PairLocks.use_db = False + PairLocks.timeframe = timeframe + Trade.use_db = False + + +def enable_database_use() -> None: + """ + Cleanup function to restore database usage. + """ + PairLocks.use_db = True + PairLocks.timeframe = '' + Trade.use_db = True From 3caf964c46ad3f8dcfcc2a7086271241e9efd72f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Jan 2024 20:01:56 +0100 Subject: [PATCH 2/5] Add NoDbContext context manager --- freqtrade/persistence/__init__.py | 3 ++- freqtrade/persistence/usedb_context.py | 11 +++++++++++ tests/persistence/test_db_context.py | 24 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/persistence/test_db_context.py diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 595090540..6205174a7 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -4,4 +4,5 @@ from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade -from freqtrade.persistence.usedb_context import disable_database_use, enable_database_use +from freqtrade.persistence.usedb_context import (FtNoDBContext, disable_database_use, + enable_database_use) diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py index 4d8c9e539..6fffd2fb5 100644 --- a/freqtrade/persistence/usedb_context.py +++ b/freqtrade/persistence/usedb_context.py @@ -20,3 +20,14 @@ def enable_database_use() -> None: PairLocks.use_db = True PairLocks.timeframe = '' Trade.use_db = True + + +class FtNoDBContext: + def __init__(self, timeframe: str = ''): + self.timeframe = timeframe + + def __enter__(self): + disable_database_use(self.timeframe) + + def __exit__(self, exc_type, exc_val, exc_tb): + enable_database_use() diff --git a/tests/persistence/test_db_context.py b/tests/persistence/test_db_context.py new file mode 100644 index 000000000..5145268f9 --- /dev/null +++ b/tests/persistence/test_db_context.py @@ -0,0 +1,24 @@ +import pytest + +from freqtrade.persistence import FtNoDBContext, PairLocks, Trade + + +@pytest.mark.parametrize('timeframe', ['', '5m', '1d']) +def test_FtNoDBContext(timeframe): + assert Trade.use_db is True + assert PairLocks.use_db is True + assert PairLocks.timeframe == '' + + with FtNoDBContext(timeframe): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == timeframe + + with FtNoDBContext(): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == '' + + assert Trade.use_db is True + assert PairLocks.use_db is True + From 883f50854419843d2de9cd0b6a9cff3563db32d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Jan 2024 20:08:23 +0100 Subject: [PATCH 3/5] Add NoDBContext to pairlist evaluation functions --- freqtrade/commands/pairlist_commands.py | 12 ++++++----- .../rpc/api_server/api_background_tasks.py | 21 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index a815cd5f3..b1df7e98b 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ + from freqtrade.persistence import FtNoDBContext from freqtrade.plugins.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) @@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: if not quote_currencies: quote_currencies = [config.get('stake_currency')] results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist + with FtNoDBContext(): + for curr in quote_currencies: + config['stake_currency'] = curr + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist for curr, pairlist in results.items(): if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index c13fa31e4..04e98c609 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -7,6 +7,7 @@ from fastapi.exceptions import HTTPException from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException +from freqtrade.persistence import FtNoDBContext from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, ExchangeModePayloadMixin, PairListsPayload, PairListsResponse, WhitelistEvaluateResponse) @@ -57,16 +58,16 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.jobs[job_id]['is_running'] = True from freqtrade.plugins.pairlistmanager import PairListManager - - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.jobs[job_id]['result'] = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - ApiBG.jobs[job_id]['status'] = 'success' + with FtNoDBContext(): + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' except (OperationalException, Exception) as e: logger.exception(e) ApiBG.jobs[job_id]['error'] = str(e) From 3f27c2792a7eafcb816725aa373396b63306a39f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Jan 2024 20:12:57 +0100 Subject: [PATCH 4/5] Remove trailing newline --- tests/persistence/test_db_context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/persistence/test_db_context.py b/tests/persistence/test_db_context.py index 5145268f9..67dcd10bb 100644 --- a/tests/persistence/test_db_context.py +++ b/tests/persistence/test_db_context.py @@ -21,4 +21,3 @@ def test_FtNoDBContext(timeframe): assert Trade.use_db is True assert PairLocks.use_db is True - From cb9fa734d1e6db602d910cb936feeea580a1973c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 10 Jan 2024 20:36:10 +0100 Subject: [PATCH 5/5] Improve test resiliance --- tests/persistence/test_db_context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/persistence/test_db_context.py b/tests/persistence/test_db_context.py index 67dcd10bb..690006219 100644 --- a/tests/persistence/test_db_context.py +++ b/tests/persistence/test_db_context.py @@ -5,6 +5,7 @@ from freqtrade.persistence import FtNoDBContext, PairLocks, Trade @pytest.mark.parametrize('timeframe', ['', '5m', '1d']) def test_FtNoDBContext(timeframe): + PairLocks.timeframe = '' assert Trade.use_db is True assert PairLocks.use_db is True assert PairLocks.timeframe == ''