diff --git a/docs/utils.md b/docs/utils.md index eb675442f..cb77ca449 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -999,3 +999,36 @@ Common arguments: Path to userdata directory. ``` +### Backtest lookahead bias checker +#### Summary +Checks a given strategy for look ahead bias +Look ahead bias means that the backtest uses data from future candles thereby not making it viable beyond backtesting +and producing false hopes for the one backtesting. + +#### Introduction: +Many strategies - without the programmer knowing - have fallen prey to look ahead bias. + +Any backtest will populate the full dataframe including all time stamps at the beginning. +If the programmer is not careful or oblivious how things work internally +(which sometimes can be really hard to find out) then it will just look into the future making the strategy amazing +but not realistic. + +The tool is made to try to verify the validity in the form of the aforementioned look ahead bias. + +#### How does the command work? +It will not look at the strategy or any contents itself but instead will run multiple backtests +by using precisely cut timeranges and analyzing the results each time, comparing to the full timerange. + +At first, it starts a backtest over the whole duration +and then repeats backtests from the same starting point to the respective points to watch. +In addition, it analyzes the dataframes form the overall backtest to the cut ones. + +At the end it will return a result-table in terminal. + +Hint: +If an entry or exit condition is only triggered rarely or the timerange was chosen +so only a few entry conditions are met +then the bias checker is unable to catch the biased entry or exit condition. +In the end it only checks which entry and exit signals have been triggered. + +---Flow chart here for better understanding--- diff --git a/freqtrade/commands/strategy_utils_commands.py b/freqtrade/commands/strategy_utils_commands.py index 417d98ad5..ab31cfa82 100755 --- a/freqtrade/commands/strategy_utils_commands.py +++ b/freqtrade/commands/strategy_utils_commands.py @@ -91,6 +91,12 @@ def start_backtest_lookahead_bias_checker(args: Dict[str, Any]) -> None: for filtered_strategy_obj in filtered_strategy_objs: bias_checker_instances.append( initialize_single_lookahead_bias_checker(filtered_strategy_obj, config, args)) + elif 'strategy' in args and args['strategy'] is not None: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == args['strategy']: + bias_checker_instances.append( + initialize_single_lookahead_bias_checker(strategy_obj, config, args)) + break else: processed_locations = set() for strategy_obj in strategy_objs: diff --git a/freqtrade/strategy/backtest_lookahead_bias_checker.py b/freqtrade/strategy/backtest_lookahead_bias_checker.py index 98b82e209..2e5ef4165 100755 --- a/freqtrade/strategy/backtest_lookahead_bias_checker.py +++ b/freqtrade/strategy/backtest_lookahead_bias_checker.py @@ -1,4 +1,6 @@ import copy +import pathlib +import shutil from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -45,8 +47,11 @@ class BacktestLookaheadBiasChecker: self.current_analysis = None self.local_config = None self.full_varHolder = None + self.entry_varHolder = None self.exit_varHolder = None + self.entry_varHolders = [] + self.exit_varHolders = [] self.backtesting = None self.minimum_trade_amount = None self.targeted_trade_amount = None @@ -105,29 +110,36 @@ class BacktestLookaheadBiasChecker: if cut_df_cut.shape[0] != 0: compare_df = full_df_cut.compare(cut_df_cut) - # skippedColumns = ["date", "open", "high", "low", "close", "volume"] - for col_name, values in compare_df.items(): - col_idx = compare_df.columns.get_loc(col_name) - compare_df_row = compare_df.iloc[0] - # compare_df now comprises tuples with [1] having either 'self' or 'other' - if 'other' in col_name[1]: - continue - self_value = compare_df_row[col_idx] - other_value = compare_df_row[col_idx + 1] + if compare_df.shape[0] > 0: + for col_name, values in compare_df.items(): + col_idx = compare_df.columns.get_loc(col_name) + compare_df_row = compare_df.iloc[0] + # compare_df now comprises tuples with [1] having either 'self' or 'other' + if 'other' in col_name[1]: + continue + self_value = compare_df_row[col_idx] + other_value = compare_df_row[col_idx + 1] - # output differences - if self_value != other_value: + # output differences + if self_value != other_value: - if not self.current_analysis.false_indicators.__contains__(col_name[0]): - self.current_analysis.false_indicators.append(col_name[0]) - print(f"=> found look ahead bias in indicator {col_name[0]}. " + - f"{str(self_value)} != {str(other_value)}") + if not self.current_analysis.false_indicators.__contains__(col_name[0]): + self.current_analysis.false_indicators.append(col_name[0]) + print(f"=> found look ahead bias in indicator {col_name[0]}. " + + f"{str(self_value)} != {str(other_value)}") def prepare_data(self, varHolder, pairs_to_load): + + # purge previous data + abs_folder_path = pathlib.Path("user_data/models/uniqe-id").resolve() + # remove folder and its contents + if pathlib.Path.exists(abs_folder_path): + shutil.rmtree(abs_folder_path) + prepare_data_config = copy.deepcopy(self.local_config) prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varHolder.from_dt)) + "-" + str(self.dt_to_timestamp(varHolder.to_dt))) - prepare_data_config['pairs'] = pairs_to_load + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load self.backtesting = Backtesting(prepare_data_config) self.backtesting._set_strategy(self.backtesting.strategylist[0]) @@ -137,9 +149,6 @@ class BacktestLookaheadBiasChecker: varHolder.indicators = self.backtesting.strategy.advise_all_indicators(varHolder.data) varHolder.result = self.get_result(self.backtesting, varHolder.indicators) - def update_output_file(self): - pass - def start(self, config, strategy_obj: dict, args) -> None: # deepcopy so we can change the pairs for the 2ndary runs @@ -195,6 +204,8 @@ class BacktestLookaheadBiasChecker: self.entry_varHolder = VarHolder() self.exit_varHolder = VarHolder() + self.entry_varHolders.append(self.entry_varHolder) + self.exit_varHolders.append(self.exit_varHolder) self.entry_varHolder.from_dt = self.full_varHolder.from_dt self.entry_varHolder.compared_dt = result_row['open_date'] @@ -224,6 +235,8 @@ class BacktestLookaheadBiasChecker: self.exit_varHolder.result, "close_date", self.exit_varHolder.compared_dt): self.current_analysis.false_exit_signals += 1 + if len(self.entry_varHolders) >= 10: + pass # check if the indicators themselves contain biased data self.analyze_indicators(self.full_varHolder, self.entry_varHolder, result_row['pair']) self.analyze_indicators(self.full_varHolder, self.exit_varHolder, result_row['pair'])