From a5f5293bc86a0a9560c8ac2d2ae5f19c58feb0a3 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 11:23:02 +0200 Subject: [PATCH 01/10] added logger-output when something is skipped or aborted --- freqtrade/optimize/lookahead_analysis.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index dcc1088b3..f8f9c7d55 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -251,9 +251,26 @@ class LookaheadAnalysis: # starting from the same datetime to avoid miss-reports of bias for idx, result_row in self.full_varHolder.result['results'].iterrows(): if self.current_analysis.total_signals == self.targeted_trade_amount: + logger.info(f"Found targeted trade amount = {self.targeted_trade_amount} signals.") break + if found_signals < self.minimum_trade_amount: + logger.info(f"only found {found_signals} " + f"which is smaller than " + f"minimum trade amount = {self.minimum_trade_amount}. " + f"Exiting this lookahead-analysis") + return None + if "force_exit" in result_row['exit_reason']: + logger.info("found force-exit, skipping this one to avoid a false-positive.") + continue + self.analyze_row(idx, result_row) + if len(self.entry_varHolders) < self.minimum_trade_amount: + logger.info(f"only found {found_signals} after skipping forced exits " + f"which is smaller than " + f"minimum trade amount = {self.minimum_trade_amount}. " + f"Exiting this lookahead-analysis") + # Restore verbosity, so it's not too quiet for the next strategy restore_verbosity_for_bias_tester() # check and report signals From a33be8a349b64067d9509dd783008450fd864027 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 13:48:54 +0200 Subject: [PATCH 02/10] added dummy-varholders in case a not-last-trade is force-exit and else the indexes would shift ruining the analysis and making debugging easier (since the same ID will always be the same ID again) --- freqtrade/optimize/lookahead_analysis.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index f8f9c7d55..e87c3c7de 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -214,6 +214,7 @@ class LookaheadAnalysis: self.entry_varHolders[idx].result, "open_date", self.entry_varHolders[idx].compared_dt): + # logger.info(f"found lookahead-bias in trade {self.entry_varHolders[idx][]} {idx}") self.current_analysis.false_entry_signals += 1 # register if buy or sell signal is broken @@ -261,6 +262,11 @@ class LookaheadAnalysis: return None if "force_exit" in result_row['exit_reason']: logger.info("found force-exit, skipping this one to avoid a false-positive.") + + # just to keep the IDs of both full, entry and exit varholders the same + # to achieve a better debugging experience + self.entry_varHolders.append(VarHolder()) + self.exit_varHolders.append(VarHolder()) continue self.analyze_row(idx, result_row) From 1ab357dc32b07ed933019810007436f07351d11f Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 15:29:25 +0200 Subject: [PATCH 03/10] added mentioning which pair + timerange + idx is biased for visibility and debugging purposes --- freqtrade/optimize/lookahead_analysis.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index e87c3c7de..af3988066 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -141,7 +141,7 @@ class LookaheadAnalysis: shutil.rmtree(path_to_current_identifier) prepare_data_config = deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + " - " + str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load @@ -209,13 +209,16 @@ class LookaheadAnalysis: # fill entry_varHolder and exit_varHolder self.fill_entry_and_exit_varHolders(result_row) + # this will trigger a logger-message + buy_or_sell_biased: bool = False + # register if buy signal is broken if not self.report_signal( self.entry_varHolders[idx].result, "open_date", self.entry_varHolders[idx].compared_dt): - # logger.info(f"found lookahead-bias in trade {self.entry_varHolders[idx][]} {idx}") self.current_analysis.false_entry_signals += 1 + buy_or_sell_biased = True # register if buy or sell signal is broken if not self.report_signal( @@ -223,6 +226,13 @@ class LookaheadAnalysis: "close_date", self.exit_varHolders[idx].compared_dt): self.current_analysis.false_exit_signals += 1 + buy_or_sell_biased = True + + if buy_or_sell_biased: + logger.info(f"found lookahead-bias in trade " + f"pair: {result_row['pair']}, " + f"timerange:{result_row['open_date']}-{result_row['close_date']}, " + f"idx: {idx}") # check if the indicators themselves contain biased data self.analyze_indicators(self.full_varHolder, self.entry_varHolders[idx], result_row['pair']) @@ -261,7 +271,9 @@ class LookaheadAnalysis: f"Exiting this lookahead-analysis") return None if "force_exit" in result_row['exit_reason']: - logger.info("found force-exit, skipping this one to avoid a false-positive.") + logger.info("found force-exit in pair: {result_row['pair']}, " + f"timerange:{result_row['open_date']}-{result_row['close_date']}, " + f"idx: {idx}, skipping this one to avoid a false-positive.") # just to keep the IDs of both full, entry and exit varholders the same # to achieve a better debugging experience From ad428aa9b08d9ecaf33ef7594093cb6cc31d7e44 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 19:50:12 +0200 Subject: [PATCH 04/10] added stake_amount to a fixed 10k value. In a combination with a wallet size of 1 billion it should never be able to run out of money avoiding false-positives of some users who just wanted to test a strategy without actually checking how the stake_amount-variable should be used in combination with the strategy-function custom_stake_amount reason: some strategies demand a custom_stake_amount of 1$ demanding a very large wallet-size (which already was set previously) Others start with 100% of a slot size and subdivide the base-orders and safety-orders down to finish at 100% of a slot-size and use unlimited stake_amount. Edited docs to reflect that change too --- docs/lookahead-analysis.md | 3 +++ freqtrade/optimize/lookahead_analysis_helpers.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index 9d57de779..c6eaa3e3c 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -22,6 +22,9 @@ It also supports the lookahead-analysis of freqai strategies. - `--cache` is forced to "none". - `--max-open-trades` is forced to be at least equal to the number of pairs. - `--dry-run-wallet` is forced to be basically infinite. +- `--stake-amount` is forced to be 10 k. + +Those are set to avoid users accidentally generating false positives. ## Lookahead-analysis command reference diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 702eee774..7b1158fc8 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -136,6 +136,12 @@ class LookaheadAnalysisSubFunctions: logger.info('Dry run wallet was not set to 1 billion, pushing it up there ' 'just to avoid false positives') config['dry_run_wallet'] = min_dry_run_wallet + # fix stake_amount to 10k. + # in a combination with a wallet size of 1 billion it should always be able to trade + # no matter if they use custom_stake_amount as a small percentage of wallet size + # or fixate custom_stake_amount to a certain value. + logger.info('fixing stake_amount to 10.000') + config['stake_amount'] = 10000 # enforce cache to be 'none', shift it to 'none' if not already # (since the default value is 'day') From e4b488cb8472fdb834dd03d025902313f84d2f44 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 20:05:29 +0200 Subject: [PATCH 05/10] added stake_amount to a fixed 10k value. In a combination with a wallet size of 1 billion it should never be able to run out of money avoiding false-positives of some users who just wanted to test a strategy without actually checking how the stake_amount-variable should be used in combination with the strategy-function custom_stake_amount. reason: some strategies demand a custom_stake_amount of 1$ demanding a very large wallet-size (which already was set previously) Others start with 100% of a slot size and subdivide the base-orders and safety-orders down to finish at 100% of a slot-size and use unlimited stake_amount. Edited docs to reflect that change. --- docs/lookahead-analysis.md | 4 ++-- freqtrade/optimize/lookahead_analysis_helpers.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/lookahead-analysis.md b/docs/lookahead-analysis.md index c6eaa3e3c..e998b1b77 100644 --- a/docs/lookahead-analysis.md +++ b/docs/lookahead-analysis.md @@ -21,8 +21,8 @@ It also supports the lookahead-analysis of freqai strategies. - `--cache` is forced to "none". - `--max-open-trades` is forced to be at least equal to the number of pairs. -- `--dry-run-wallet` is forced to be basically infinite. -- `--stake-amount` is forced to be 10 k. +- `--dry-run-wallet` is forced to be basically infinite (1 billion). +- `--stake-amount` is forced to be a static 10000 (10k). Those are set to avoid users accidentally generating false positives. diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 7b1158fc8..1fd4706f5 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -136,11 +136,12 @@ class LookaheadAnalysisSubFunctions: logger.info('Dry run wallet was not set to 1 billion, pushing it up there ' 'just to avoid false positives') config['dry_run_wallet'] = min_dry_run_wallet + # fix stake_amount to 10k. # in a combination with a wallet size of 1 billion it should always be able to trade # no matter if they use custom_stake_amount as a small percentage of wallet size # or fixate custom_stake_amount to a certain value. - logger.info('fixing stake_amount to 10.000') + logger.info('fixing stake_amount to 10k') config['stake_amount'] = 10000 # enforce cache to be 'none', shift it to 'none' if not already From 5b8800ee1819a20f12b1c0c42865effd535e5978 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sun, 23 Jul 2023 20:20:15 +0200 Subject: [PATCH 06/10] didnt intend to change the timerange itself, but the logger-output of the timerange --- freqtrade/optimize/lookahead_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index af3988066..f363ae196 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -141,7 +141,7 @@ class LookaheadAnalysis: shutil.rmtree(path_to_current_identifier) prepare_data_config = deepcopy(self.local_config) - prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + " - " + + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load @@ -231,7 +231,7 @@ class LookaheadAnalysis: if buy_or_sell_biased: logger.info(f"found lookahead-bias in trade " f"pair: {result_row['pair']}, " - f"timerange:{result_row['open_date']}-{result_row['close_date']}, " + f"timerange:{result_row['open_date']} - {result_row['close_date']}, " f"idx: {idx}") # check if the indicators themselves contain biased data From 25602ceac35811e2d5377ccf8e08c84f1425b112 Mon Sep 17 00:00:00 2001 From: hippocritical Date: Sat, 5 Aug 2023 08:24:47 +0200 Subject: [PATCH 07/10] Added a fixed fee to 0.02 (any fixed value would suffice) since kucoin dynamically decides which pair gets which amount of fees and thereby producing false-positives upon verifying the entries/exits. Added a check for timerange being set. --- freqtrade/optimize/lookahead_analysis_helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 1fd4706f5..654ff93d2 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -137,6 +137,16 @@ class LookaheadAnalysisSubFunctions: 'just to avoid false positives') config['dry_run_wallet'] = min_dry_run_wallet + if 'fee' not in config or config['fee'] != 0.02: + logger.info('fee was not set to a fixed value of 0.02. ') + config['fee'] = 0.02 + + if 'timerange' not in config: + # setting a timerange is enforced here + raise OperationalException( + "Please set a timerange. " + "Usually a few months are enough depending on your needs and strategy." + ) # fix stake_amount to 10k. # in a combination with a wallet size of 1 billion it should always be able to trade # no matter if they use custom_stake_amount as a small percentage of wallet size From 4a62ebbf9369904200499c7a23bb5732f6586900 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Aug 2023 18:36:09 +0200 Subject: [PATCH 08/10] Don't hardcode fee, but use fee from the very first iteration. --- freqtrade/optimize/lookahead_analysis.py | 8 +++++++- freqtrade/optimize/lookahead_analysis_helpers.py | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index f363ae196..0543fcde7 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -48,6 +48,7 @@ class LookaheadAnalysis: self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] self.exchange: Optional[Any] = None + self._fee = None # pull variables the scope of the lookahead_analysis-instance self.local_config = deepcopy(config) @@ -145,8 +146,13 @@ class LookaheadAnalysis: str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + if self._fee: + # Don't re-calculate fee per pair, as fee might differ per pair. + prepare_data_config['fee'] = self._fee + backtesting = Backtesting(prepare_data_config, self.exchange) self.exchange = backtesting.exchange + self._fee = backtesting.fee backtesting._set_strategy(backtesting.strategylist[0]) varholder.data, varholder.timerange = backtesting.load_bt_data() @@ -198,7 +204,7 @@ class LookaheadAnalysis: self.prepare_data(exit_varHolder, [result_row['pair']]) # now we analyze a full trade of full_varholder and look for analyze its bias - def analyze_row(self, idx, result_row): + def analyze_row(self, idx: int, result_row): # if force-sold, ignore this signal since here it will unconditionally exit. if result_row.close_date == self.dt_to_timestamp(self.full_varHolder.to_dt): return diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 654ff93d2..422026780 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -137,10 +137,6 @@ class LookaheadAnalysisSubFunctions: 'just to avoid false positives') config['dry_run_wallet'] = min_dry_run_wallet - if 'fee' not in config or config['fee'] != 0.02: - logger.info('fee was not set to a fixed value of 0.02. ') - config['fee'] = 0.02 - if 'timerange' not in config: # setting a timerange is enforced here raise OperationalException( From b93464403997d5e272a07b66474d2d1eb4526b45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Aug 2023 18:36:20 +0200 Subject: [PATCH 09/10] Fix tests, explicitly test for missing timerange --- tests/optimize/test_lookahead_analysis.py | 24 ++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_lookahead_analysis.py b/tests/optimize/test_lookahead_analysis.py index 3c6a5ad6d..decc4706d 100644 --- a/tests/optimize/test_lookahead_analysis.py +++ b/tests/optimize/test_lookahead_analysis.py @@ -17,6 +17,8 @@ from tests.conftest import EXMS, get_args, log_has_re, patch_exchange def lookahead_conf(default_conf_usdt): default_conf_usdt['minimum_trade_amount'] = 10 default_conf_usdt['targeted_trade_amount'] = 20 + default_conf_usdt['timerange'] = '20220101-20220501' + default_conf_usdt['strategy_path'] = str( Path(__file__).parent.parent / "strategy/strats/lookahead_bias") default_conf_usdt['strategy'] = 'strategy_test_v3_with_lookahead_bias' @@ -43,7 +45,9 @@ def test_start_lookahead_analysis(mocker): "--pairs", "UNITTEST/BTC", "--max-open-trades", - "1" + "1", + "--timerange", + "20220101-20220201" ] pargs = get_args(args) pargs['config'] = None @@ -72,6 +76,24 @@ def test_start_lookahead_analysis(mocker): match=r"Targeted trade amount can't be smaller than minimum trade amount.*"): start_lookahead_analysis(pargs) + # Missing timerange + args = [ + "lookahead-analysis", + "--strategy", + "strategy_test_v3_with_lookahead_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats/lookahead_bias"), + "--pairs", + "UNITTEST/BTC", + "--max-open-trades", + "1", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_lookahead_analysis(pargs) + def test_lookahead_helper_invalid_config(lookahead_conf) -> None: conf = deepcopy(lookahead_conf) From 05e1828617d27d71c1d2d84967aca2b2f92d3404 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Aug 2023 20:26:08 +0200 Subject: [PATCH 10/10] Improve Fee check --- freqtrade/optimize/lookahead_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 0543fcde7..80418da95 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -146,7 +146,7 @@ class LookaheadAnalysis: str(self.dt_to_timestamp(varholder.to_dt))) prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load - if self._fee: + if self._fee is not None: # Don't re-calculate fee per pair, as fee might differ per pair. prepare_data_config['fee'] = self._fee