From 08dffc95d877ebcb605835b186ce4924c409f209 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 13 Sep 2023 11:58:28 +0900 Subject: [PATCH 01/12] fix wording --- freqtrade/optimize/recursive_analysis_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 0b353edfc..8bae36dd2 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -39,12 +39,19 @@ class RecursiveAnalysisSubFunctions: @staticmethod def calculate_config_overrides(config: Config): + if 'timerange' not in config: + # setting a timerange is enforced here + raise OperationalException( + "Please set a timerange. " + "A timerange of 20 candles are enough for recursive analysis." + ) + if config.get('backtest_cache') is None: config['backtest_cache'] = 'none' elif config['backtest_cache'] != 'none': logger.info(f"backtest_cache = " f"{config['backtest_cache']} detected. " - f"Inside lookahead-analysis it is enforced to be 'none'. " + f"Inside recursive-analysis it is enforced to be 'none'. " f"Changed it to 'none'") config['backtest_cache'] = 'none' return config @@ -57,7 +64,7 @@ class RecursiveAnalysisSubFunctions: current_instance = RecursiveAnalysis(config, strategy_obj) current_instance.start() elapsed = time.perf_counter() - start - logger.info(f"Checking recursive and lookahead bias of indicators " + logger.info(f"Checking recursive and indicator-only lookahead bias of indicators " f"of {Path(strategy_obj['location']).name} " f"took {elapsed:.0f} seconds.") return current_instance From 979e485f24dfda6fba3e1a07950a6adc71a131b3 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 20 Sep 2023 20:53:34 +0900 Subject: [PATCH 02/12] initial doc --- docs/recursive-analysis.md | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 docs/recursive-analysis.md diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md new file mode 100644 index 000000000..3a8b3017c --- /dev/null +++ b/docs/recursive-analysis.md @@ -0,0 +1,96 @@ +# Lookahead analysis + +This page explains how to validate your strategy in terms of recursive formula issue. + +First of all, what is recursive formula? Recursive formula is a formula that defines any term of a sequence in terms of its preceding term(s). Example of a recursive formula is an = an-1 + b. + +Second question is why is it matter for Freqtrade? It matters because in backtesting, the bot will get full data of the pairs according to the timerange specified. But in dry/live run, the bot will have limited amounts of data, limited by what each exchanges gives. + +For example, let's say that I want to calculate a very basic indicator called `steps`. The first row's value is always 0, while the following rows' values are equal to the value of the previous row's plus 1. If I were to calculate it using latest 1000 candles, then the `steps` value of first row is 0, and the `steps` value at last closed candle is 999. + +But what if I only calculate based of latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result. + +Recursive-analysis requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, +head over to the [Data Downloading](data-download.md) section of the documentation. + +This command is built upon backtesting since it internally chains backtests to prepare different lenghts of data and calculate indicators based of each of the prepared data. +This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. + +You can use commands of [Backtesting](backtesting.md). + +- `--cache` is forced to "none". +- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. +- It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. + +Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits. It will only check whether there is any difference in indicators' value if the end of the data are moved. + +## Recursive-analysis command reference + +``` +usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] + [--recursive-strategy-search] + [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] + [-p PAIRS [PAIRS ...]] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] + [--export {none,trades,signals}] + [--export-filename PATH] + [--breakdown {day,week,month} [{day,week,month} ...]] + [--cache {none,day,week,month}] + [--freqai-backtest-live-models] + [--minimum-trade-amount INT] + [--targeted-trade-amount INT] + [--lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME] + +options: + --minimum-trade-amount INT + Minimum trade amount for lookahead-analysis + --targeted-trade-amount INT + Targeted trade amount for lookahead analysis + --lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME + Use this csv-filename to store lookahead-analysis- + results +``` + +!!! Note "" + The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands. + +### Summary + +Checks a given strategy for look ahead bias via lookahead-analysis +Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting +and producing false hopes for the one backtesting. + +### Introduction + +Many strategies - without the programmer knowing - have fallen prey to look ahead bias. + +Any backtest will populate the full dataframe including all time stamps at the beginning. +If the programmer is not careful or oblivious how things work internally +(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing +but not realistic. + +This command is made to try to verify the validity in the form of the aforementioned look ahead bias. + +### How does the command work? + +It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. +After the backtest ran, it will look if the `minimum-trade-amount` is met +and if not cancel the lookahead-analysis for this strategy. + +After setting the baseline it will then do additional runs for every entry and exit separately. +When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias. +After all signals have been verified or falsified a result-table will be generated for the user to see. + +### Caveats + +- `lookahead-analysis` can only verify / falsify the trades it calculated and verified. +If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified. +This could lead to a false-negative (the strategy will then be reported as non-biased). +- `lookahead-analysis` has access to everything that backtesting has too. +Please don't provoke any configs like enabling position stacking. +If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. From 37fa186c556a783c2f2719040adce648a81acfd6 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Wed, 20 Sep 2023 22:43:01 +0900 Subject: [PATCH 03/12] remove 1 column --- docs/recursive-analysis.md | 21 ------------------- .../optimize/recursive_analysis_helpers.py | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 3a8b3017c..86b358830 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -16,8 +16,6 @@ head over to the [Data Downloading](data-download.md) section of the documentati This command is built upon backtesting since it internally chains backtests to prepare different lenghts of data and calculate indicators based of each of the prepared data. This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. -You can use commands of [Backtesting](backtesting.md). - - `--cache` is forced to "none". - Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. - It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. @@ -36,29 +34,10 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--timerange TIMERANGE] [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] [-p PAIRS [PAIRS ...]] - [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export {none,trades,signals}] - [--export-filename PATH] - [--breakdown {day,week,month} [{day,week,month} ...]] - [--cache {none,day,week,month}] [--freqai-backtest-live-models] - [--minimum-trade-amount INT] - [--targeted-trade-amount INT] - [--lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME] -options: - --minimum-trade-amount INT - Minimum trade amount for lookahead-analysis - --targeted-trade-amount INT - Targeted trade amount for lookahead analysis - --lookahead-analysis-exportfilename LOOKAHEAD_ANALYSIS_EXPORTFILENAME - Use this csv-filename to store lookahead-analysis- - results ``` -!!! Note "" - The above Output was reduced to options `lookahead-analysis` adds on top of regular backtesting commands. - ### Summary Checks a given strategy for look ahead bias via lookahead-analysis diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 8bae36dd2..167ea5386 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -19,7 +19,7 @@ class RecursiveAnalysisSubFunctions: config: Dict[str, Any], recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle - headers = ['strategy', 'indicators'] + headers = ['indicators'] for candle in startups: headers.append(candle) @@ -27,7 +27,7 @@ class RecursiveAnalysisSubFunctions: for inst in recursive_instances: if len(inst.dict_recursive) > 0: for indicator, values in inst.dict_recursive.items(): - temp_data = [inst.strategy_obj['name'], indicator] + temp_data = [indicator] for candle in startups: temp_data.append(values.get(int(candle), '-')) data.append(temp_data) From e77b9de89e799b1598ca7ab0548d91dc967cb5fe Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 10:51:12 +0900 Subject: [PATCH 04/12] fix docs --- docs/recursive-analysis.md | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 86b358830..10c1fe077 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -17,10 +17,10 @@ This command is built upon backtesting since it internally chains backtests to p This is done by not looking at the strategy itself - but at the value of the indicators it returned. After multiple backtests are done to calculate the indicators of different startup candles value, the values of last rows are compared to see hoe much differences are they compared to the base backtest. - `--cache` is forced to "none". -- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis the first pair in the whitelist. +- Since we are only looking at indicators' value, using more than one pair is redundant. It is recommended to set the pair used in the command using `-p` flag, preferably using pair with high price, such as BTC or ETH, to avoid having rounding issue that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. - It's recommended to set a long timerange (at least consist of 5000 candles), so that the initial backtest that going to be used as benchmark have very small or no recursive issue at all. For example, for a 5m timeframe, timerange of 5000 candles would be equal to 18 days. -Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits. It will only check whether there is any difference in indicators' value if the end of the data are moved. +Beside recursive formula check, this command also going to do a simple lookahead bias check on the indicators' value only. It won't replace [Lookahead-analysis](lookahead-analysis.md), since this check won't check the difference in trades' entries and exits, which is the important effect of lookahead bias. It will only check whether there is any lookahead bias in indicators if the end of the data are moved. ## Recursive-analysis command reference @@ -40,36 +40,16 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] ### Summary -Checks a given strategy for look ahead bias via lookahead-analysis -Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting -and producing false hopes for the one backtesting. - -### Introduction - -Many strategies - without the programmer knowing - have fallen prey to look ahead bias. - -Any backtest will populate the full dataframe including all time stamps at the beginning. -If the programmer is not careful or oblivious how things work internally -(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing -but not realistic. - -This command is made to try to verify the validity in the form of the aforementioned look ahead bias. +Checks a given strategy for recursive formula issue via recursive-analysis. +Recursive formula issue means that the indicator's calculation don't have enough data for its calculation to produce correct value. ### How does the command work? -It will start with a backtest of all pairs to generate a baseline for indicators and entries/exits. -After the backtest ran, it will look if the `minimum-trade-amount` is met -and if not cancel the lookahead-analysis for this strategy. - -After setting the baseline it will then do additional runs for every entry and exit separately. -When a verification-backtest is done, it will compare the indicators as the signal (either entry or exit) and report the bias. -After all signals have been verified or falsified a result-table will be generated for the user to see. +It will start with a backtest using the supplied timerange to generate a baseline for indicators' value. +After setting the baseline it will then do additional runs for each different startup candles. +When the additional runs are done, it will compare the indicators at the last rows and report the differences in a table. ### Caveats -- `lookahead-analysis` can only verify / falsify the trades it calculated and verified. -If the strategy has many different signals / signal types, it's up to you to select appropriate parameters to ensure that all signals have triggered at least once. Not triggered signals will not have been verified. -This could lead to a false-negative (the strategy will then be reported as non-biased). -- `lookahead-analysis` has access to everything that backtesting has too. -Please don't provoke any configs like enabling position stacking. -If you decide to do so, then make doubly sure that you won't ever run out of `max_open_trades` amount and neither leftover money in your wallet. +- `recursive-analysis` will only calculate and compare the indicators' value at the last row. If there are any differences, the table will only tell you the percentage differences. Whether it has any real impact on your entries and exits isn't checked. +- The ideal scenario is to have your indicators have no difference at all despite the startup candle being varied. But in reality, some of publicly-available formulas are using recursive formula. So the goal isn't to have zero differences, but to have the differences low enough to make sure they won't have any real impact on trading decisions. From 32b0098ec11985b352cfb48a38d1e34522e65806 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 11:08:47 +0900 Subject: [PATCH 05/12] fix example in the docs, increasing startup to 400 on ema100 --- docs/recursive-analysis.md | 2 +- docs/strategy-customization.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md index 10c1fe077..341493f7e 100644 --- a/docs/recursive-analysis.md +++ b/docs/recursive-analysis.md @@ -1,4 +1,4 @@ -# Lookahead analysis +# Recursive analysis This page explains how to validate your strategy in terms of recursive formula issue. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 005127715..e23c3cc41 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -168,10 +168,12 @@ Most indicators have an instable startup period, in which they are either not av To account for this, the strategy can be assigned the `startup_candle_count` attribute. This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators. -In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. +You can use [recursive-analysis](recursive-analysis.md) to check and find the correct `startup_candle_count` to be used. + +In this example strategy, this should be set to 400 (`startup_candle_count = 400`), since the minimum needed history for ema100 calculation to make sure the value is correct is 400 candles. ``` python - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=400) ``` By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. @@ -193,11 +195,11 @@ Let's try to backtest 1 month (January 2019) of 5m candles using an example stra freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m ``` -Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00. +Assuming `startup_candle_count` is set to 400, backtesting knows it needs 400 candles to generate valid buy signals. It will load data from `20190101 - (400 * 5m)` - which is ~2018-12-30 11:40:00. If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note - If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-02 09:20:00. ### Entry signal rules From d465fcffd5a70d46b70fb36937619776f9bbd46c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 11:13:06 +0900 Subject: [PATCH 06/12] change startup cande in sample strat --- freqtrade/templates/sample_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index fd81570fe..65a6e440e 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -77,7 +77,7 @@ class SampleStrategy(IStrategy): exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 30 + startup_candle_count: int = 170 # Optional order type mapping. order_types = { From 28e43a486783855933da7121fbc6f63e4be3b3ce Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 21 Sep 2023 14:00:17 +0900 Subject: [PATCH 07/12] initial test --- tests/optimize/test_recursive_analysis.py | 370 ++++++++++++++++++ .../strategy_test_v3_recursive_issue.py | 43 ++ 2 files changed, 413 insertions(+) create mode 100644 tests/optimize/test_recursive_analysis.py create mode 100644 tests/strategy/strats/strategy_test_v3_recursive_issue.py diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py new file mode 100644 index 000000000..021d3eb72 --- /dev/null +++ b/tests/optimize/test_recursive_analysis.py @@ -0,0 +1,370 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +from copy import deepcopy +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pytest + +from freqtrade.commands.optimize_commands import start_recursive_analysis +from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.recursive_analysis import Analysis, RecursiveAnalysis +from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions +from tests.conftest import EXMS, get_args, log_has_re, patch_exchange + + +@pytest.fixture +def recursive_conf(default_conf_usdt): + default_conf_usdt['timerange'] = '20220101-20220501' + + default_conf_usdt['strategy_path'] = str( + Path(__file__).parent.parent / "strategy/strats") + default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue' + default_conf_usdt['pairs'] = ['UNITTEST/USDT'] + return default_conf_usdt + + +def test_start_recursive_analysis(mocker): + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_recursive_issue", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC", + "--timerange", + "20220101-20220201" + ] + pargs = get_args(args) + pargs['config'] = None + + start_recursive_analysis(pargs) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + + # Test invalid config + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--targeted-trade-amount", + "10", + "--minimum-trade-amount", + "20", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): + start_lookahead_analysis(pargs) + + # Missing timerange + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--pairs", + "UNITTEST/BTC", + "--max-open-trades", + "1", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_lookahead_analysis(pargs) + + +def test_lookahead_helper_invalid_config(lookahead_conf) -> None: + conf = deepcopy(lookahead_conf) + conf['targeted_trade_amount'] = 10 + conf['minimum_trade_amount'] = 40 + with pytest.raises(OperationalException, + match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): + LookaheadAnalysisSubFunctions.start(conf) + + +def test_lookahead_helper_no_strategy_defined(lookahead_conf): + conf = deepcopy(lookahead_conf) + conf['pairs'] = ['UNITTEST/USDT'] + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + LookaheadAnalysisSubFunctions.start(conf) + + +def test_lookahead_helper_start(lookahead_conf, mocker) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', + initialize_single_lookahead_analysis=single_mock, + text_table_lookahead_analysis_instances=text_table_mock, + ) + LookaheadAnalysisSubFunctions.start(lookahead_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + text_table_mock.reset_mock() + + +def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): + analysis = Analysis() + analysis.has_bias = True + analysis.total_signals = 5 + analysis.false_entry_signals = 4 + analysis.false_exit_signals = 3 + + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + + # check row contents for a try that has too few signals + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2].__contains__('too few trades') + assert len(data[0]) == 3 + + # now check for an error which occured after enough trades + analysis.total_signals = 12 + analysis.false_entry_signals = 11 + analysis.false_exit_signals = 10 + instance = LookaheadAnalysis(lookahead_conf, strategy_obj) + instance.current_analysis = analysis + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + assert data[0][2].__contains__("error") + + # edit it into not showing an error + instance.failed_bias_check = False + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' + assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' + assert data[0][2] # True + assert data[0][3] == 12 + assert data[0][4] == 11 + assert data[0][5] == 10 + assert data[0][6] == '' + + analysis.false_indicators.append('falseIndicator1') + analysis.false_indicators.append('falseIndicator2') + table, headers, data = (LookaheadAnalysisSubFunctions. + text_table_lookahead_analysis_instances(lookahead_conf, [instance])) + + assert data[0][6] == 'falseIndicator1, falseIndicator2' + + # check amount of returning rows + assert len(data) == 1 + + # check amount of multiple rows + table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( + lookahead_conf, [instance, instance, instance])) + assert len(data) == 3 + + +def test_lookahead_helper_export_to_csv(lookahead_conf): + import pandas as pd + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + # just to be sure the test won't fail: remove file if exists for some reason + # (repeat this at the end once again to clean up) + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() + + # before we can start we have to delete the + + # 1st check: create a new file and verify its contents + analysis1 = Analysis() + analysis1.has_bias = True + analysis1.total_signals = 12 + analysis1.false_entry_signals = 11 + analysis1.false_exit_signals = 10 + analysis1.false_indicators.append('falseIndicator1') + analysis1.false_indicators.append('falseIndicator2') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj1 = { + 'name': "strat1", + 'location': Path("file1.py"), + } + + instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) + instance1.failed_bias_check = False + instance1.current_analysis = analysis1 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) + saved_data1 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + expected_values1 = [ + [ + 'file1.py', 'strat1', True, + 12, 11, 10, + "falseIndicator1,falseIndicator2" + ], + ] + expected_columns = ['filename', 'strategy', 'has_bias', + 'total_signals', 'biased_entry_signals', 'biased_exit_signals', + 'biased_indicators'] + expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) + + assert Path(lookahead_conf['lookahead_analysis_exportfilename']).exists() + assert expected_data1.equals(saved_data1) + + # 2nd check: update the same strategy (which internally changed or is being retested) + expected_values2 = [ + [ + 'file1.py', 'strat1', False, + 22, 21, 20, + "falseIndicator3,falseIndicator4" + ], + ] + expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) + + analysis2 = Analysis() + analysis2.has_bias = False + analysis2.total_signals = 22 + analysis2.false_entry_signals = 21 + analysis2.false_exit_signals = 20 + analysis2.false_indicators.append('falseIndicator3') + analysis2.false_indicators.append('falseIndicator4') + + strategy_obj2 = { + 'name': "strat1", + 'location': Path("file1.py"), + } + + instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) + instance2.failed_bias_check = False + instance2.current_analysis = analysis2 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) + saved_data2 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + + assert expected_data2.equals(saved_data2) + + # 3rd check: now we add a new row to an already existing file + expected_values3 = [ + [ + 'file1.py', 'strat1', False, + 22, 21, 20, + "falseIndicator3,falseIndicator4" + ], + [ + 'file3.py', 'strat3', True, + 32, 31, 30, "falseIndicator5,falseIndicator6" + ], + ] + + expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) + + analysis3 = Analysis() + analysis3.has_bias = True + analysis3.total_signals = 32 + analysis3.false_entry_signals = 31 + analysis3.false_exit_signals = 30 + analysis3.false_indicators.append('falseIndicator5') + analysis3.false_indicators.append('falseIndicator6') + lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" + + strategy_obj3 = { + 'name': "strat3", + 'location': Path("file3.py"), + } + + instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) + instance3.failed_bias_check = False + instance3.current_analysis = analysis3 + + LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) + saved_data3 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) + assert expected_data3.equals(saved_data3) + + # remove csv file after the test is done + if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): + Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() + + +def test_initialize_single_lookahead_analysis(lookahead_conf, mocker, caplog): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + mocker.patch(f'{EXMS}.get_fee', return_value=0.0) + mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) + mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] + + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '20180119-20180122' + start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_with_lookahead_bias", + 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") + } + + instance = LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( + lookahead_conf, strategy_obj) + assert log_has_re(r"Bias test of .* started\.", caplog) + assert start_mock.call_count == 1 + + assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" + + +@pytest.mark.parametrize('scenario', [ + 'no_bias', 'bias1' +]) +def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + lookahead_conf['pairs'] = ['UNITTEST/USDT'] + + lookahead_conf['timeframe'] = '5m' + lookahead_conf['timerange'] = '20180119-20180122' + lookahead_conf['startup_candle'] = [100] + + # Patch scenario Parameter to allow for easy selection + mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', + return_value={ + 'params': { + "buy": { + "scenario": scenario + } + } + }) + + strategy_obj = {'name': "strategy_test_v3_recursive_issue"} + instance = RecursiveAnalysis(lookahead_conf, strategy_obj) + instance.start() + # Assert init correct + assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + + # check non-biased strategy + if scenario == "no_bias": + assert not instance.current_analysis.has_bias + # check biased strategy + elif scenario == "bias1": + assert instance.current_analysis.has_bias diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py new file mode 100644 index 000000000..c4ddc6a1b --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -0,0 +1,43 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +from pandas import DataFrame +from technical.indicators import ichimoku + +from freqtrade.strategy import IStrategy +from freqtrade.strategy.parameters import CategoricalParameter + +import talib.abstract as ta + +class strategy_test_v3_recursive_issue(IStrategy): + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + scenario = CategoricalParameter(['no_bias', 'bias1'], default='bias1', space="buy") + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 100 + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # bias is introduced here + if self.scenario.value == 'no_bias': + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + else: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe From 08b94a20778d808af03fd107108b86155437fd4f Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 14:21:54 +0900 Subject: [PATCH 08/12] 1 test --- tests/optimize/test_recursive_analysis.py | 527 +++++++++++----------- 1 file changed, 264 insertions(+), 263 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 021d3eb72..458c07829 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -24,312 +24,312 @@ def recursive_conf(default_conf_usdt): return default_conf_usdt -def test_start_recursive_analysis(mocker): - single_mock = MagicMock() - text_table_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', - initialize_single_recursive_analysis=single_mock, - text_table_recursive_analysis_instances=text_table_mock, - ) - args = [ - "recursive-analysis", - "--strategy", - "strategy_test_v3_recursive_issue", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats"), - "--pairs", - "UNITTEST/BTC", - "--timerange", - "20220101-20220201" - ] - pargs = get_args(args) - pargs['config'] = None +# def test_start_recursive_analysis(mocker): +# single_mock = MagicMock() +# text_table_mock = MagicMock() +# mocker.patch.multiple( +# 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', +# initialize_single_recursive_analysis=single_mock, +# text_table_recursive_analysis_instances=text_table_mock, +# ) +# args = [ +# "recursive-analysis", +# "--strategy", +# "strategy_test_v3_recursive_issue", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats"), +# "--pairs", +# "UNITTEST/BTC", +# "--timerange", +# "20220101-20220201" +# ] +# pargs = get_args(args) +# pargs['config'] = None - start_recursive_analysis(pargs) - assert single_mock.call_count == 1 - assert text_table_mock.call_count == 1 +# start_recursive_analysis(pargs) +# assert single_mock.call_count == 1 +# assert text_table_mock.call_count == 1 - single_mock.reset_mock() +# single_mock.reset_mock() - # Test invalid config - args = [ - "lookahead-analysis", - "--strategy", - "strategy_test_v3_with_lookahead_bias", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), - "--targeted-trade-amount", - "10", - "--minimum-trade-amount", - "20", - ] - pargs = get_args(args) - pargs['config'] = None - with pytest.raises(OperationalException, - match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): - start_lookahead_analysis(pargs) +# # Test invalid config +# args = [ +# "lookahead-analysis", +# "--strategy", +# "strategy_test_v3_with_lookahead_bias", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# "--targeted-trade-amount", +# "10", +# "--minimum-trade-amount", +# "20", +# ] +# pargs = get_args(args) +# pargs['config'] = None +# with pytest.raises(OperationalException, +# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): +# start_lookahead_analysis(pargs) - # Missing timerange - args = [ - "lookahead-analysis", - "--strategy", - "strategy_test_v3_with_lookahead_bias", - "--strategy-path", - str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), - "--pairs", - "UNITTEST/BTC", - "--max-open-trades", - "1", - ] - pargs = get_args(args) - pargs['config'] = None - with pytest.raises(OperationalException, - match=r"Please set a timerange\..*"): - start_lookahead_analysis(pargs) +# # Missing timerange +# args = [ +# "lookahead-analysis", +# "--strategy", +# "strategy_test_v3_with_lookahead_bias", +# "--strategy-path", +# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# "--pairs", +# "UNITTEST/BTC", +# "--max-open-trades", +# "1", +# ] +# pargs = get_args(args) +# pargs['config'] = None +# with pytest.raises(OperationalException, +# match=r"Please set a timerange\..*"): +# start_lookahead_analysis(pargs) -def test_lookahead_helper_invalid_config(lookahead_conf) -> None: - conf = deepcopy(lookahead_conf) - conf['targeted_trade_amount'] = 10 - conf['minimum_trade_amount'] = 40 - with pytest.raises(OperationalException, - match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): - LookaheadAnalysisSubFunctions.start(conf) +# def test_lookahead_helper_invalid_config(recursive_conf) -> None: +# conf = deepcopy(recursive_conf) +# conf['targeted_trade_amount'] = 10 +# conf['minimum_trade_amount'] = 40 +# with pytest.raises(OperationalException, +# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): +# RecursiveAnalysisSubFunctions.start(conf) -def test_lookahead_helper_no_strategy_defined(lookahead_conf): - conf = deepcopy(lookahead_conf) - conf['pairs'] = ['UNITTEST/USDT'] - del conf['strategy'] - with pytest.raises(OperationalException, - match=r"No Strategy specified"): - LookaheadAnalysisSubFunctions.start(conf) +# def test_lookahead_helper_no_strategy_defined(recursive_conf): +# conf = deepcopy(recursive_conf) +# conf['pairs'] = ['UNITTEST/USDT'] +# del conf['strategy'] +# with pytest.raises(OperationalException, +# match=r"No Strategy specified"): +# RecursiveAnalysisSubFunctions.start(conf) -def test_lookahead_helper_start(lookahead_conf, mocker) -> None: - single_mock = MagicMock() - text_table_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.optimize.lookahead_analysis_helpers.LookaheadAnalysisSubFunctions', - initialize_single_lookahead_analysis=single_mock, - text_table_lookahead_analysis_instances=text_table_mock, - ) - LookaheadAnalysisSubFunctions.start(lookahead_conf) - assert single_mock.call_count == 1 - assert text_table_mock.call_count == 1 +# def test_lookahead_helper_start(recursive_conf, mocker) -> None: +# single_mock = MagicMock() +# text_table_mock = MagicMock() +# mocker.patch.multiple( +# 'freqtrade.optimize.lookahead_analysis_helpers.RecursiveAnalysisSubFunctions', +# initialize_single_lookahead_analysis=single_mock, +# text_table_lookahead_analysis_instances=text_table_mock, +# ) +# RecursiveAnalysisSubFunctions.start(recursive_conf) +# assert single_mock.call_count == 1 +# assert text_table_mock.call_count == 1 - single_mock.reset_mock() - text_table_mock.reset_mock() +# single_mock.reset_mock() +# text_table_mock.reset_mock() -def test_lookahead_helper_text_table_lookahead_analysis_instances(lookahead_conf): - analysis = Analysis() - analysis.has_bias = True - analysis.total_signals = 5 - analysis.false_entry_signals = 4 - analysis.false_exit_signals = 3 +# def test_lookahead_helper_text_table_lookahead_analysis_instances(recursive_conf): +# analysis = Analysis() +# analysis.has_bias = True +# analysis.total_signals = 5 +# analysis.false_entry_signals = 4 +# analysis.false_exit_signals = 3 - strategy_obj = { - 'name': "strategy_test_v3_with_lookahead_bias", - 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") - } +# strategy_obj = { +# 'name': "strategy_test_v3_with_lookahead_bias", +# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") +# } - instance = LookaheadAnalysis(lookahead_conf, strategy_obj) - instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) +# instance = LookaheadAnalysis(recursive_conf, strategy_obj) +# instance.current_analysis = analysis +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - # check row contents for a try that has too few signals - assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' - assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' - assert data[0][2].__contains__('too few trades') - assert len(data[0]) == 3 +# # check row contents for a try that has too few signals +# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' +# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' +# assert data[0][2].__contains__('too few trades') +# assert len(data[0]) == 3 - # now check for an error which occured after enough trades - analysis.total_signals = 12 - analysis.false_entry_signals = 11 - analysis.false_exit_signals = 10 - instance = LookaheadAnalysis(lookahead_conf, strategy_obj) - instance.current_analysis = analysis - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) - assert data[0][2].__contains__("error") +# # now check for an error which occured after enough trades +# analysis.total_signals = 12 +# analysis.false_entry_signals = 11 +# analysis.false_exit_signals = 10 +# instance = LookaheadAnalysis(recursive_conf, strategy_obj) +# instance.current_analysis = analysis +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) +# assert data[0][2].__contains__("error") - # edit it into not showing an error - instance.failed_bias_check = False - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) - assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' - assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' - assert data[0][2] # True - assert data[0][3] == 12 - assert data[0][4] == 11 - assert data[0][5] == 10 - assert data[0][6] == '' +# # edit it into not showing an error +# instance.failed_bias_check = False +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) +# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' +# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' +# assert data[0][2] # True +# assert data[0][3] == 12 +# assert data[0][4] == 11 +# assert data[0][5] == 10 +# assert data[0][6] == '' - analysis.false_indicators.append('falseIndicator1') - analysis.false_indicators.append('falseIndicator2') - table, headers, data = (LookaheadAnalysisSubFunctions. - text_table_lookahead_analysis_instances(lookahead_conf, [instance])) +# analysis.false_indicators.append('falseIndicator1') +# analysis.false_indicators.append('falseIndicator2') +# table, headers, data = (RecursiveAnalysisSubFunctions. +# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - assert data[0][6] == 'falseIndicator1, falseIndicator2' +# assert data[0][6] == 'falseIndicator1, falseIndicator2' - # check amount of returning rows - assert len(data) == 1 +# # check amount of returning rows +# assert len(data) == 1 - # check amount of multiple rows - table, headers, data = (LookaheadAnalysisSubFunctions.text_table_lookahead_analysis_instances( - lookahead_conf, [instance, instance, instance])) - assert len(data) == 3 +# # check amount of multiple rows +# table, headers, data = (RecursiveAnalysisSubFunctions.text_table_lookahead_analysis_instances( +# recursive_conf, [instance, instance, instance])) +# assert len(data) == 3 -def test_lookahead_helper_export_to_csv(lookahead_conf): - import pandas as pd - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# def test_lookahead_helper_export_to_csv(recursive_conf): +# import pandas as pd +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - # just to be sure the test won't fail: remove file if exists for some reason - # (repeat this at the end once again to clean up) - if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): - Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() +# # just to be sure the test won't fail: remove file if exists for some reason +# # (repeat this at the end once again to clean up) +# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): +# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() - # before we can start we have to delete the +# # before we can start we have to delete the - # 1st check: create a new file and verify its contents - analysis1 = Analysis() - analysis1.has_bias = True - analysis1.total_signals = 12 - analysis1.false_entry_signals = 11 - analysis1.false_exit_signals = 10 - analysis1.false_indicators.append('falseIndicator1') - analysis1.false_indicators.append('falseIndicator2') - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# # 1st check: create a new file and verify its contents +# analysis1 = Analysis() +# analysis1.has_bias = True +# analysis1.total_signals = 12 +# analysis1.false_entry_signals = 11 +# analysis1.false_exit_signals = 10 +# analysis1.false_indicators.append('falseIndicator1') +# analysis1.false_indicators.append('falseIndicator2') +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - strategy_obj1 = { - 'name': "strat1", - 'location': Path("file1.py"), - } +# strategy_obj1 = { +# 'name': "strat1", +# 'location': Path("file1.py"), +# } - instance1 = LookaheadAnalysis(lookahead_conf, strategy_obj1) - instance1.failed_bias_check = False - instance1.current_analysis = analysis1 +# instance1 = LookaheadAnalysis(recursive_conf, strategy_obj1) +# instance1.failed_bias_check = False +# instance1.current_analysis = analysis1 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance1]) - saved_data1 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance1]) +# saved_data1 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - expected_values1 = [ - [ - 'file1.py', 'strat1', True, - 12, 11, 10, - "falseIndicator1,falseIndicator2" - ], - ] - expected_columns = ['filename', 'strategy', 'has_bias', - 'total_signals', 'biased_entry_signals', 'biased_exit_signals', - 'biased_indicators'] - expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) +# expected_values1 = [ +# [ +# 'file1.py', 'strat1', True, +# 12, 11, 10, +# "falseIndicator1,falseIndicator2" +# ], +# ] +# expected_columns = ['filename', 'strategy', 'has_bias', +# 'total_signals', 'biased_entry_signals', 'biased_exit_signals', +# 'biased_indicators'] +# expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) - assert Path(lookahead_conf['lookahead_analysis_exportfilename']).exists() - assert expected_data1.equals(saved_data1) +# assert Path(recursive_conf['lookahead_analysis_exportfilename']).exists() +# assert expected_data1.equals(saved_data1) - # 2nd check: update the same strategy (which internally changed or is being retested) - expected_values2 = [ - [ - 'file1.py', 'strat1', False, - 22, 21, 20, - "falseIndicator3,falseIndicator4" - ], - ] - expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) +# # 2nd check: update the same strategy (which internally changed or is being retested) +# expected_values2 = [ +# [ +# 'file1.py', 'strat1', False, +# 22, 21, 20, +# "falseIndicator3,falseIndicator4" +# ], +# ] +# expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) - analysis2 = Analysis() - analysis2.has_bias = False - analysis2.total_signals = 22 - analysis2.false_entry_signals = 21 - analysis2.false_exit_signals = 20 - analysis2.false_indicators.append('falseIndicator3') - analysis2.false_indicators.append('falseIndicator4') +# analysis2 = Analysis() +# analysis2.has_bias = False +# analysis2.total_signals = 22 +# analysis2.false_entry_signals = 21 +# analysis2.false_exit_signals = 20 +# analysis2.false_indicators.append('falseIndicator3') +# analysis2.false_indicators.append('falseIndicator4') - strategy_obj2 = { - 'name': "strat1", - 'location': Path("file1.py"), - } +# strategy_obj2 = { +# 'name': "strat1", +# 'location': Path("file1.py"), +# } - instance2 = LookaheadAnalysis(lookahead_conf, strategy_obj2) - instance2.failed_bias_check = False - instance2.current_analysis = analysis2 +# instance2 = LookaheadAnalysis(recursive_conf, strategy_obj2) +# instance2.failed_bias_check = False +# instance2.current_analysis = analysis2 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance2]) - saved_data2 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance2]) +# saved_data2 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - assert expected_data2.equals(saved_data2) +# assert expected_data2.equals(saved_data2) - # 3rd check: now we add a new row to an already existing file - expected_values3 = [ - [ - 'file1.py', 'strat1', False, - 22, 21, 20, - "falseIndicator3,falseIndicator4" - ], - [ - 'file3.py', 'strat3', True, - 32, 31, 30, "falseIndicator5,falseIndicator6" - ], - ] +# # 3rd check: now we add a new row to an already existing file +# expected_values3 = [ +# [ +# 'file1.py', 'strat1', False, +# 22, 21, 20, +# "falseIndicator3,falseIndicator4" +# ], +# [ +# 'file3.py', 'strat3', True, +# 32, 31, 30, "falseIndicator5,falseIndicator6" +# ], +# ] - expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) +# expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) - analysis3 = Analysis() - analysis3.has_bias = True - analysis3.total_signals = 32 - analysis3.false_entry_signals = 31 - analysis3.false_exit_signals = 30 - analysis3.false_indicators.append('falseIndicator5') - analysis3.false_indicators.append('falseIndicator6') - lookahead_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +# analysis3 = Analysis() +# analysis3.has_bias = True +# analysis3.total_signals = 32 +# analysis3.false_entry_signals = 31 +# analysis3.false_exit_signals = 30 +# analysis3.false_indicators.append('falseIndicator5') +# analysis3.false_indicators.append('falseIndicator6') +# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - strategy_obj3 = { - 'name': "strat3", - 'location': Path("file3.py"), - } +# strategy_obj3 = { +# 'name': "strat3", +# 'location': Path("file3.py"), +# } - instance3 = LookaheadAnalysis(lookahead_conf, strategy_obj3) - instance3.failed_bias_check = False - instance3.current_analysis = analysis3 +# instance3 = LookaheadAnalysis(recursive_conf, strategy_obj3) +# instance3.failed_bias_check = False +# instance3.current_analysis = analysis3 - LookaheadAnalysisSubFunctions.export_to_csv(lookahead_conf, [instance3]) - saved_data3 = pd.read_csv(lookahead_conf['lookahead_analysis_exportfilename']) - assert expected_data3.equals(saved_data3) +# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance3]) +# saved_data3 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) +# assert expected_data3.equals(saved_data3) - # remove csv file after the test is done - if Path(lookahead_conf['lookahead_analysis_exportfilename']).exists(): - Path(lookahead_conf['lookahead_analysis_exportfilename']).unlink() +# # remove csv file after the test is done +# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): +# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() -def test_initialize_single_lookahead_analysis(lookahead_conf, mocker, caplog): - mocker.patch('freqtrade.data.history.get_timerange', get_timerange) - mocker.patch(f'{EXMS}.get_fee', return_value=0.0) - mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) - mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) - patch_exchange(mocker) - mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', - PropertyMock(return_value=['UNITTEST/BTC'])) - lookahead_conf['pairs'] = ['UNITTEST/USDT'] +# def test_initialize_single_lookahead_analysis(recursive_conf, mocker, caplog): +# mocker.patch('freqtrade.data.history.get_timerange', get_timerange) +# mocker.patch(f'{EXMS}.get_fee', return_value=0.0) +# mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) +# mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) +# patch_exchange(mocker) +# mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', +# PropertyMock(return_value=['UNITTEST/BTC'])) +# recursive_conf['pairs'] = ['UNITTEST/USDT'] - lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '20180119-20180122' - start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') - strategy_obj = { - 'name': "strategy_test_v3_with_lookahead_bias", - 'location': Path(lookahead_conf['strategy_path'], f"{lookahead_conf['strategy']}.py") - } +# recursive_conf['timeframe'] = '5m' +# recursive_conf['timerange'] = '20180119-20180122' +# start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') +# strategy_obj = { +# 'name': "strategy_test_v3_with_lookahead_bias", +# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") +# } - instance = LookaheadAnalysisSubFunctions.initialize_single_lookahead_analysis( - lookahead_conf, strategy_obj) - assert log_has_re(r"Bias test of .* started\.", caplog) - assert start_mock.call_count == 1 +# instance = RecursiveAnalysisSubFunctions.initialize_single_lookahead_analysis( +# recursive_conf, strategy_obj) +# assert log_has_re(r"Bias test of .* started\.", caplog) +# assert start_mock.call_count == 1 - assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" +# assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" @pytest.mark.parametrize('scenario', [ @@ -340,11 +340,11 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - lookahead_conf['pairs'] = ['UNITTEST/USDT'] + recursive_conf['pairs'] = ['UNITTEST/USDT'] - lookahead_conf['timeframe'] = '5m' - lookahead_conf['timerange'] = '20180119-20180122' - lookahead_conf['startup_candle'] = [100] + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + recursive_conf['startup_candle'] = [100] # Patch scenario Parameter to allow for easy selection mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', @@ -357,14 +357,15 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: }) strategy_obj = {'name': "strategy_test_v3_recursive_issue"} - instance = RecursiveAnalysis(lookahead_conf, strategy_obj) + instance = RecursiveAnalysis(recursive_conf, strategy_obj) instance.start() # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + diff_pct = float(instance.dict_recursive['rsi'][100].replace("%", "")) # check non-biased strategy if scenario == "no_bias": - assert not instance.current_analysis.has_bias + assert diff_pct < 0.01 # check biased strategy elif scenario == "bias1": - assert instance.current_analysis.has_bias + assert diff_pct >= 0.01 From b9e9f82503a9baf5b0e290598b2f3bae0b8b2d46 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 16:45:43 +0900 Subject: [PATCH 09/12] first test done --- freqtrade/optimize/recursive_analysis.py | 3 +++ tests/optimize/test_recursive_analysis.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py index 599fc4dda..45c2a457c 100644 --- a/freqtrade/optimize/recursive_analysis.py +++ b/freqtrade/optimize/recursive_analysis.py @@ -129,6 +129,7 @@ class RecursiveAnalysis(BaseAnalysis): varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) def fill_partial_varholder(self, start_date, startup_candle): + logger.info(f"Calculating indicators using startup candle of {startup_candle}.") partial_varHolder = VarHolder() partial_varHolder.from_dt = start_date @@ -142,6 +143,8 @@ class RecursiveAnalysis(BaseAnalysis): self.partial_varHolder_array.append(partial_varHolder) def fill_partial_varholder_lookahead(self, end_date): + logger.info("Calculating indicators to test lookahead on indicators.") + partial_varHolder = VarHolder() partial_varHolder.from_dt = self.full_varHolder.from_dt diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 458c07829..4d5ddc63a 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -8,7 +8,7 @@ import pytest from freqtrade.commands.optimize_commands import start_recursive_analysis from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException -from freqtrade.optimize.recursive_analysis import Analysis, RecursiveAnalysis +from freqtrade.optimize.recursive_analysis import RecursiveAnalysis from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions from tests.conftest import EXMS, get_args, log_has_re, patch_exchange @@ -340,7 +340,7 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: patch_exchange(mocker) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - recursive_conf['pairs'] = ['UNITTEST/USDT'] + recursive_conf['pairs'] = ['UNITTEST/BTC'] recursive_conf['timeframe'] = '5m' recursive_conf['timerange'] = '20180119-20180122' @@ -362,7 +362,7 @@ def test_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: # Assert init correct assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) - diff_pct = float(instance.dict_recursive['rsi'][100].replace("%", "")) + diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) # check non-biased strategy if scenario == "no_bias": assert diff_pct < 0.01 From 89d47ffd8fdca2372a231f1399f2e67065b72463 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:47:51 +0900 Subject: [PATCH 10/12] 4 more tests --- .../optimize/recursive_analysis_helpers.py | 3 +- tests/optimize/test_recursive_analysis.py | 286 ++++-------------- 2 files changed, 66 insertions(+), 223 deletions(-) diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py index 167ea5386..b33a24cf4 100644 --- a/freqtrade/optimize/recursive_analysis_helpers.py +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -16,7 +16,6 @@ class RecursiveAnalysisSubFunctions: @staticmethod def text_table_recursive_analysis_instances( - config: Dict[str, Any], recursive_instances: List[RecursiveAnalysis]): startups = recursive_instances[0]._startup_candle headers = ['indicators'] @@ -99,7 +98,7 @@ class RecursiveAnalysisSubFunctions: # report the results if RecursiveAnalysis_instances: RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( - config, RecursiveAnalysis_instances) + RecursiveAnalysis_instances) else: logger.error("There were no strategies specified neither through " "--strategy nor through " diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index 4d5ddc63a..d06cbdb90 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -21,6 +21,7 @@ def recursive_conf(default_conf_usdt): Path(__file__).parent.parent / "strategy/strats") default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue' default_conf_usdt['pairs'] = ['UNITTEST/USDT'] + default_conf_usdt['startup_candle'] = [100] return default_conf_usdt @@ -54,11 +55,11 @@ def recursive_conf(default_conf_usdt): # # Test invalid config # args = [ -# "lookahead-analysis", +# "recursive-analysis", # "--strategy", -# "strategy_test_v3_with_lookahead_bias", +# "strategy_test_v3_with_recursive_bias", # "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), # "--targeted-trade-amount", # "10", # "--minimum-trade-amount", @@ -68,15 +69,15 @@ def recursive_conf(default_conf_usdt): # pargs['config'] = None # with pytest.raises(OperationalException, # match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# start_lookahead_analysis(pargs) +# start_recursive_analysis(pargs) # # Missing timerange # args = [ -# "lookahead-analysis", +# "recursive-analysis", # "--strategy", -# "strategy_test_v3_with_lookahead_bias", +# "strategy_test_v3_with_recursive_bias", # "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), +# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), # "--pairs", # "UNITTEST/BTC", # "--max-open-trades", @@ -86,10 +87,10 @@ def recursive_conf(default_conf_usdt): # pargs['config'] = None # with pytest.raises(OperationalException, # match=r"Please set a timerange\..*"): -# start_lookahead_analysis(pargs) +# start_recursive_analysis(pargs) -# def test_lookahead_helper_invalid_config(recursive_conf) -> None: +# def test_recursive_helper_invalid_config(recursive_conf) -> None: # conf = deepcopy(recursive_conf) # conf['targeted_trade_amount'] = 10 # conf['minimum_trade_amount'] = 40 @@ -98,7 +99,7 @@ def recursive_conf(default_conf_usdt): # RecursiveAnalysisSubFunctions.start(conf) -# def test_lookahead_helper_no_strategy_defined(recursive_conf): +# def test_recursive_helper_no_strategy_defined(recursive_conf): # conf = deepcopy(recursive_conf) # conf['pairs'] = ['UNITTEST/USDT'] # del conf['strategy'] @@ -107,229 +108,72 @@ def recursive_conf(default_conf_usdt): # RecursiveAnalysisSubFunctions.start(conf) -# def test_lookahead_helper_start(recursive_conf, mocker) -> None: -# single_mock = MagicMock() -# text_table_mock = MagicMock() -# mocker.patch.multiple( -# 'freqtrade.optimize.lookahead_analysis_helpers.RecursiveAnalysisSubFunctions', -# initialize_single_lookahead_analysis=single_mock, -# text_table_lookahead_analysis_instances=text_table_mock, -# ) -# RecursiveAnalysisSubFunctions.start(recursive_conf) -# assert single_mock.call_count == 1 -# assert text_table_mock.call_count == 1 +def test_recursive_helper_start(recursive_conf, mocker) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + RecursiveAnalysisSubFunctions.start(recursive_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 -# single_mock.reset_mock() -# text_table_mock.reset_mock() + single_mock.reset_mock() + text_table_mock.reset_mock() -# def test_lookahead_helper_text_table_lookahead_analysis_instances(recursive_conf): -# analysis = Analysis() -# analysis.has_bias = True -# analysis.total_signals = 5 -# analysis.false_entry_signals = 4 -# analysis.false_exit_signals = 3 +def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf): + dict_diff = dict() + dict_diff['rsi'] = {} + dict_diff['rsi'][100] = "0.078%" -# strategy_obj = { -# 'name': "strategy_test_v3_with_lookahead_bias", -# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") -# } + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } -# instance = LookaheadAnalysis(recursive_conf, strategy_obj) -# instance.current_analysis = analysis -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) -# # check row contents for a try that has too few signals -# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' -# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' -# assert data[0][2].__contains__('too few trades') -# assert len(data[0]) == 3 + # check row contents for a try that has too few signals + assert data[0][0] == 'rsi' + assert data[0][1] == '0.078%' + assert len(data[0]) == 2 -# # now check for an error which occured after enough trades -# analysis.total_signals = 12 -# analysis.false_entry_signals = 11 -# analysis.false_exit_signals = 10 -# instance = LookaheadAnalysis(recursive_conf, strategy_obj) -# instance.current_analysis = analysis -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) -# assert data[0][2].__contains__("error") - -# # edit it into not showing an error -# instance.failed_bias_check = False -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) -# assert data[0][0] == 'strategy_test_v3_with_lookahead_bias.py' -# assert data[0][1] == 'strategy_test_v3_with_lookahead_bias' -# assert data[0][2] # True -# assert data[0][3] == 12 -# assert data[0][4] == 11 -# assert data[0][5] == 10 -# assert data[0][6] == '' - -# analysis.false_indicators.append('falseIndicator1') -# analysis.false_indicators.append('falseIndicator2') -# table, headers, data = (RecursiveAnalysisSubFunctions. -# text_table_lookahead_analysis_instances(recursive_conf, [instance])) - -# assert data[0][6] == 'falseIndicator1, falseIndicator2' - -# # check amount of returning rows -# assert len(data) == 1 - -# # check amount of multiple rows -# table, headers, data = (RecursiveAnalysisSubFunctions.text_table_lookahead_analysis_instances( -# recursive_conf, [instance, instance, instance])) -# assert len(data) == 3 + # now check when there is no issue + dict_diff = dict() + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) + assert len(data) == 0 -# def test_lookahead_helper_export_to_csv(recursive_conf): -# import pandas as pd -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" +def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + recursive_conf['pairs'] = ['UNITTEST/BTC'] -# # just to be sure the test won't fail: remove file if exists for some reason -# # (repeat this at the end once again to clean up) -# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): -# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + start_mock = mocker.patch('freqtrade.optimize.recursive_analysis.RecursiveAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } -# # before we can start we have to delete the + instance = RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis( + recursive_conf, strategy_obj) + assert log_has_re(r"Recursive test of .* started\.", caplog) + assert start_mock.call_count == 1 -# # 1st check: create a new file and verify its contents -# analysis1 = Analysis() -# analysis1.has_bias = True -# analysis1.total_signals = 12 -# analysis1.false_entry_signals = 11 -# analysis1.false_exit_signals = 10 -# analysis1.false_indicators.append('falseIndicator1') -# analysis1.false_indicators.append('falseIndicator2') -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - -# strategy_obj1 = { -# 'name': "strat1", -# 'location': Path("file1.py"), -# } - -# instance1 = LookaheadAnalysis(recursive_conf, strategy_obj1) -# instance1.failed_bias_check = False -# instance1.current_analysis = analysis1 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance1]) -# saved_data1 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - -# expected_values1 = [ -# [ -# 'file1.py', 'strat1', True, -# 12, 11, 10, -# "falseIndicator1,falseIndicator2" -# ], -# ] -# expected_columns = ['filename', 'strategy', 'has_bias', -# 'total_signals', 'biased_entry_signals', 'biased_exit_signals', -# 'biased_indicators'] -# expected_data1 = pd.DataFrame(expected_values1, columns=expected_columns) - -# assert Path(recursive_conf['lookahead_analysis_exportfilename']).exists() -# assert expected_data1.equals(saved_data1) - -# # 2nd check: update the same strategy (which internally changed or is being retested) -# expected_values2 = [ -# [ -# 'file1.py', 'strat1', False, -# 22, 21, 20, -# "falseIndicator3,falseIndicator4" -# ], -# ] -# expected_data2 = pd.DataFrame(expected_values2, columns=expected_columns) - -# analysis2 = Analysis() -# analysis2.has_bias = False -# analysis2.total_signals = 22 -# analysis2.false_entry_signals = 21 -# analysis2.false_exit_signals = 20 -# analysis2.false_indicators.append('falseIndicator3') -# analysis2.false_indicators.append('falseIndicator4') - -# strategy_obj2 = { -# 'name': "strat1", -# 'location': Path("file1.py"), -# } - -# instance2 = LookaheadAnalysis(recursive_conf, strategy_obj2) -# instance2.failed_bias_check = False -# instance2.current_analysis = analysis2 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance2]) -# saved_data2 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) - -# assert expected_data2.equals(saved_data2) - -# # 3rd check: now we add a new row to an already existing file -# expected_values3 = [ -# [ -# 'file1.py', 'strat1', False, -# 22, 21, 20, -# "falseIndicator3,falseIndicator4" -# ], -# [ -# 'file3.py', 'strat3', True, -# 32, 31, 30, "falseIndicator5,falseIndicator6" -# ], -# ] - -# expected_data3 = pd.DataFrame(expected_values3, columns=expected_columns) - -# analysis3 = Analysis() -# analysis3.has_bias = True -# analysis3.total_signals = 32 -# analysis3.false_entry_signals = 31 -# analysis3.false_exit_signals = 30 -# analysis3.false_indicators.append('falseIndicator5') -# analysis3.false_indicators.append('falseIndicator6') -# recursive_conf['lookahead_analysis_exportfilename'] = "temp_csv_lookahead_analysis.csv" - -# strategy_obj3 = { -# 'name': "strat3", -# 'location': Path("file3.py"), -# } - -# instance3 = LookaheadAnalysis(recursive_conf, strategy_obj3) -# instance3.failed_bias_check = False -# instance3.current_analysis = analysis3 - -# RecursiveAnalysisSubFunctions.export_to_csv(recursive_conf, [instance3]) -# saved_data3 = pd.read_csv(recursive_conf['lookahead_analysis_exportfilename']) -# assert expected_data3.equals(saved_data3) - -# # remove csv file after the test is done -# if Path(recursive_conf['lookahead_analysis_exportfilename']).exists(): -# Path(recursive_conf['lookahead_analysis_exportfilename']).unlink() - - -# def test_initialize_single_lookahead_analysis(recursive_conf, mocker, caplog): -# mocker.patch('freqtrade.data.history.get_timerange', get_timerange) -# mocker.patch(f'{EXMS}.get_fee', return_value=0.0) -# mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.00001) -# mocker.patch(f'{EXMS}.get_max_pair_stake_amount', return_value=float('inf')) -# patch_exchange(mocker) -# mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', -# PropertyMock(return_value=['UNITTEST/BTC'])) -# recursive_conf['pairs'] = ['UNITTEST/USDT'] - -# recursive_conf['timeframe'] = '5m' -# recursive_conf['timerange'] = '20180119-20180122' -# start_mock = mocker.patch('freqtrade.optimize.lookahead_analysis.LookaheadAnalysis.start') -# strategy_obj = { -# 'name': "strategy_test_v3_with_lookahead_bias", -# 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") -# } - -# instance = RecursiveAnalysisSubFunctions.initialize_single_lookahead_analysis( -# recursive_conf, strategy_obj) -# assert log_has_re(r"Bias test of .* started\.", caplog) -# assert start_mock.call_count == 1 - -# assert instance.strategy_obj['name'] == "strategy_test_v3_with_lookahead_bias" + assert instance.strategy_obj['name'] == "strategy_test_v3_recursive_issue" @pytest.mark.parametrize('scenario', [ From d92a6d7b737bd8356db9e46a40a29c9c1e9b764e Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:51:37 +0900 Subject: [PATCH 11/12] all tests done --- tests/optimize/test_recursive_analysis.py | 123 +++++++++------------- 1 file changed, 47 insertions(+), 76 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index d06cbdb90..cc7b63f08 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -25,87 +25,58 @@ def recursive_conf(default_conf_usdt): return default_conf_usdt -# def test_start_recursive_analysis(mocker): -# single_mock = MagicMock() -# text_table_mock = MagicMock() -# mocker.patch.multiple( -# 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', -# initialize_single_recursive_analysis=single_mock, -# text_table_recursive_analysis_instances=text_table_mock, -# ) -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_recursive_issue", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats"), -# "--pairs", -# "UNITTEST/BTC", -# "--timerange", -# "20220101-20220201" -# ] -# pargs = get_args(args) -# pargs['config'] = None +def test_start_recursive_analysis(mocker): + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_recursive_issue", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC", + "--timerange", + "20220101-20220201" + ] + pargs = get_args(args) + pargs['config'] = None -# start_recursive_analysis(pargs) -# assert single_mock.call_count == 1 -# assert text_table_mock.call_count == 1 + start_recursive_analysis(pargs) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 -# single_mock.reset_mock() + single_mock.reset_mock() -# # Test invalid config -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_with_recursive_bias", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), -# "--targeted-trade-amount", -# "10", -# "--minimum-trade-amount", -# "20", -# ] -# pargs = get_args(args) -# pargs['config'] = None -# with pytest.raises(OperationalException, -# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# start_recursive_analysis(pargs) - -# # Missing timerange -# args = [ -# "recursive-analysis", -# "--strategy", -# "strategy_test_v3_with_recursive_bias", -# "--strategy-path", -# str(Path(__file__).parent.parent / "strategy/strats/recursive_bias"), -# "--pairs", -# "UNITTEST/BTC", -# "--max-open-trades", -# "1", -# ] -# pargs = get_args(args) -# pargs['config'] = None -# with pytest.raises(OperationalException, -# match=r"Please set a timerange\..*"): -# start_recursive_analysis(pargs) + # Missing timerange + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_with_recursive_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC" + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_recursive_analysis(pargs) -# def test_recursive_helper_invalid_config(recursive_conf) -> None: -# conf = deepcopy(recursive_conf) -# conf['targeted_trade_amount'] = 10 -# conf['minimum_trade_amount'] = 40 -# with pytest.raises(OperationalException, -# match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): -# RecursiveAnalysisSubFunctions.start(conf) - - -# def test_recursive_helper_no_strategy_defined(recursive_conf): -# conf = deepcopy(recursive_conf) -# conf['pairs'] = ['UNITTEST/USDT'] -# del conf['strategy'] -# with pytest.raises(OperationalException, -# match=r"No Strategy specified"): -# RecursiveAnalysisSubFunctions.start(conf) +def test_recursive_helper_no_strategy_defined(recursive_conf): + conf = deepcopy(recursive_conf) + conf['pairs'] = ['UNITTEST/USDT'] + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + RecursiveAnalysisSubFunctions.start(conf) def test_recursive_helper_start(recursive_conf, mocker) -> None: From aba576f79f33da4e72c846fec533b56d4c6b0b07 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 21 Sep 2023 17:58:20 +0900 Subject: [PATCH 12/12] pre-commit fix --- tests/optimize/test_recursive_analysis.py | 2 +- tests/strategy/strats/strategy_test_v3_recursive_issue.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py index cc7b63f08..6560a4f26 100644 --- a/tests/optimize/test_recursive_analysis.py +++ b/tests/optimize/test_recursive_analysis.py @@ -10,7 +10,7 @@ from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.optimize.recursive_analysis import RecursiveAnalysis from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions -from tests.conftest import EXMS, get_args, log_has_re, patch_exchange +from tests.conftest import get_args, log_has_re, patch_exchange @pytest.fixture diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py index c4ddc6a1b..78a9dca61 100644 --- a/tests/strategy/strats/strategy_test_v3_recursive_issue.py +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -1,11 +1,10 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +import talib.abstract as ta from pandas import DataFrame -from technical.indicators import ichimoku from freqtrade.strategy import IStrategy from freqtrade.strategy.parameters import CategoricalParameter -import talib.abstract as ta class strategy_test_v3_recursive_issue(IStrategy): INTERFACE_VERSION = 3