From 9488e8992dea615420b4712be52774b8995bec5e Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:24:52 +0100 Subject: [PATCH 01/25] First commit for integrating buy_reasons into FT --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 14 +- freqtrade/commands/cli_options.py | 31 +++ freqtrade/configuration/configuration.py | 15 ++ freqtrade/data/entryexitanalysis.py | 258 +++++++++++++++++++++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100755 freqtrade/data/entryexitanalysis.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0e637c487..d93ed1e09 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -6,6 +6,7 @@ Contains all start-commands, subcommands and CLI Interface creation. Note: Be careful with file-scoped imports in these subfiles. as they are parsed on startup, nothing containing optional modules should be loaded. """ +from freqtrade.commands.analyze_commands import start_analysis_entries_exits from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 815e28175..4dd0141fa 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,6 +101,9 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", + "exit_reason_list", "indicator_list"] + NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", @@ -182,8 +185,9 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_backtesting, start_backtesting_show, - start_convert_data, start_convert_db, start_convert_trades, + from freqtrade.commands import (start_analysis_entries_exits, start_backtesting, + start_backtesting_show, start_convert_data, + start_convert_db, start_convert_trades, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, @@ -415,3 +419,9 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) + + # Add backtesting analysis subcommand + analysis_cmd = subparsers.add_parser('analysis', help='Analysis module.', + parents=[_common_parser, _strategy_parser]) + analysis_cmd.set_defaults(func=start_analysis_entries_exits) + self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index aac9f5713..f925bd699 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -614,4 +614,35 @@ AVAILABLE_CLI_OPTIONS = { "that do not contain any parameters."), action="store_true", ), + "analysis_groups": Arg( + "--analysis_groups", + help=("grouping output - ", + "0: simple wins/losses by enter tag, ", + "1: by enter_tag, ", + "2: by enter_tag and exit_tag, ", + "3: by pair and enter_tag, ", + "4: by pair, enter_ and exit_tag (this can get quite large)"), + nargs='?', + default="0,1,2", + ), + "enter_reason_list": Arg( + "--enter_reason_list", + help=("Comma separated list of entry signals to analyse. Default: all. ", + "e.g. 'entry_tag_a,entry_tag_b'"), + nargs='?', + default='all', + ), + "exit_reason_list": Arg( + "--exit_reason_list", + help=("Comma separated list of exit signals to analyse. Default: all. ", + "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), + nargs='?', + default='all', + ), + "indicator_list": Arg( + "--indicator_list", + help=("Comma separated list of indicators to analyse. ", + "e.g. 'close,rsi,bb_lowerband,profit_abs'"), + nargs='?', + ), } diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 96b585cd1..ea4bcace8 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -95,6 +95,8 @@ class Configuration: self._process_data_options(config) + self._process_analyze_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -433,6 +435,19 @@ class Configuration: self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}') + def _process_analyze_options(self, config: Dict[str, Any]) -> None: + self._args_to_config(config, argname='analysis_groups', + logstring='Analysis reason groups: {}') + + self._args_to_config(config, argname='enter_reason_list', + logstring='Analysis enter tag list: {}') + + self._args_to_config(config, argname='exit_reason_list', + logstring='Analysis exit tag list: {}') + + self._args_to_config(config, argname='indicator_list', + logstring='Analysis indicator list: {}') + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py new file mode 100755 index 000000000..62216d5ea --- /dev/null +++ b/freqtrade/data/entryexitanalysis.py @@ -0,0 +1,258 @@ +import joblib +import logging +import os + +from pathlib import Path +from typing import List, Optional + +import pandas as pd +from tabulate import tabulate + +from freqtrade.data.btanalysis import (load_backtest_data, get_latest_backtest_filename) +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def _load_signal_candles(backtest_dir: Path): + scpf = Path(backtest_dir, + os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + try: + scp = open(scpf, "rb") + signal_candles = joblib.load(scp) + logger.info(f"Loaded signal candles: {str(scpf)}") + except Exception as e: + logger.error("Cannot load signal candles from pickled results: ", e) + + return signal_candles + + +def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles): + analysed_trades_dict = {} + analysed_trades_dict[strategy_name] = {} + + try: + logger.info(f"Processing {strategy_name} : {len(pairlist)} pairs") + + for pair in pairlist: + if pair in signal_candles[strategy_name]: + analysed_trades_dict[strategy_name][pair] = _analyze_candles_and_indicators( + pair, + trades, + signal_candles[strategy_name][pair]) + except Exception: + pass + + return analysed_trades_dict + + +def _analyze_candles_and_indicators(pair, trades, signal_candles): + buyf = signal_candles + + if len(buyf) > 0: + buyf = buyf.set_index('date', drop=False) + trades_red = trades.loc[trades['pair'] == pair].copy() + + trades_inds = pd.DataFrame() + + if trades_red.shape[0] > 0 and buyf.shape[0] > 0: + for t, v in trades_red.open_date.items(): + allinds = buyf.loc[(buyf['date'] < v)] + if allinds.shape[0] > 0: + tmp_inds = allinds.iloc[[-1]] + + trades_red.loc[t, 'signal_date'] = tmp_inds['date'].values[0] + trades_red.loc[t, 'enter_reason'] = trades_red.loc[t, 'enter_tag'] + tmp_inds.index.rename('signal_date', inplace=True) + trades_inds = pd.concat([trades_inds, tmp_inds]) + + if 'signal_date' in trades_red: + trades_red['signal_date'] = pd.to_datetime(trades_red['signal_date'], utc=True) + trades_red.set_index('signal_date', inplace=True) + + try: + trades_red = pd.merge(trades_red, trades_inds, on='signal_date', how='outer') + except Exception as e: + print(e) + return trades_red + else: + return pd.DataFrame() + + +def _do_group_table_output(bigdf, glist): + if "0" in glist: + wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ + .groupby(['enter_reason']) \ + .agg({'profit_abs': ['sum']}) + + wins.columns = ['profit_abs_wins'] + loss = bigdf.loc[bigdf['profit_abs'] < 0] \ + .groupby(['enter_reason']) \ + .agg({'profit_abs': ['sum']}) + loss.columns = ['profit_abs_loss'] + + new = bigdf.groupby(['enter_reason']).agg({'profit_abs': [ + 'count', + lambda x: sum(x > 0), + lambda x: sum(x <= 0)]}) + + new = pd.merge(new, wins, left_index=True, right_index=True) + new = pd.merge(new, loss, left_index=True, right_index=True) + + new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) + + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]) + + new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', + 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] + + sortcols = ['total_num_buys'] + + _print_table(new, sortcols, show_index=True) + if "1" in glist: + new = bigdf.groupby(['enter_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['enter_reason', 'num_buys', 'profit_abs_sum', 'profit_abs_median', + 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', + 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "2" in glist: + new = bigdf.groupby(['enter_reason', 'exit_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "3" in glist: + new = bigdf.groupby(['pair', 'enter_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['pair', 'enter_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + if "4" in glist: + new = bigdf.groupby(['pair', 'enter_reason', 'exit_reason']) \ + .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + 'profit_ratio': ['sum', 'median', 'mean']} + ).reset_index() + new.columns = ['pair', 'enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', + 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', + 'mean_profit_pct', 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] + + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) + + +def _print_results(analysed_trades, stratname, group, + enter_reason_list, exit_reason_list, + indicator_list, columns=None): + + if columns is None: + columns = ['pair', 'open_date', 'close_date', 'profit_abs', 'enter_reason', 'exit_reason'] + + bigdf = pd.DataFrame() + for pair, trades in analysed_trades[stratname].items(): + bigdf = pd.concat([bigdf, trades], ignore_index=True) + + if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): + if group is not None: + glist = group.split(",") + _do_group_table_output(bigdf, glist) + + if enter_reason_list is not None and not enter_reason_list == "all": + enter_reason_list = enter_reason_list.split(",") + bigdf = bigdf.loc[(bigdf['enter_reason'].isin(enter_reason_list))] + + if exit_reason_list is not None and not exit_reason_list == "all": + exit_reason_list = exit_reason_list.split(",") + bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] + + if indicator_list is not None: + if indicator_list == "all": + print(bigdf) + else: + available_inds = [] + for ind in indicator_list.split(","): + if ind in bigdf: + available_inds.append(ind) + ilist = ["pair", "enter_reason", "exit_reason"] + available_inds + print(tabulate(bigdf[ilist].sort_values(['exit_reason']), + headers='keys', tablefmt='psql', showindex=False)) + else: + print(tabulate(bigdf[columns].sort_values(['pair']), + headers='keys', tablefmt='psql', showindex=False)) + else: + print("\\_ No trades to show") + + +def _print_table(df, sortcols=None, show_index=False): + if (sortcols is not None): + data = df.sort_values(sortcols) + else: + data = df + + print( + tabulate( + data, + headers='keys', + tablefmt='psql', + showindex=show_index + ) + ) + + +def process_entry_exit_reasons(backtest_dir: Path, + pairlist: List[str], + strategy_name: str, + analysis_groups: Optional[str] = "0,1,2", + enter_reason_list: Optional[str] = "all", + exit_reason_list: Optional[str] = "all", + indicator_list: Optional[str] = None): + + try: + trades = load_backtest_data(backtest_dir, strategy_name) + except ValueError as e: + raise OperationalException(e) from e + if not trades.empty: + signal_candles = _load_signal_candles(backtest_dir) + analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, + trades, signal_candles) + _print_results(analysed_trades_dict, + strategy_name, + analysis_groups, + enter_reason_list, + exit_reason_list, + indicator_list) From a1a09a802b8232b0285dbdad1d9542936cf53232 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:34:31 +0100 Subject: [PATCH 02/25] Add analyze_commands --- freqtrade/commands/analyze_commands.py | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 freqtrade/commands/analyze_commands.py diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py new file mode 100755 index 000000000..0bda9935a --- /dev/null +++ b/freqtrade/commands/analyze_commands.py @@ -0,0 +1,62 @@ +import logging +import os + +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for the entry/exit reason analysis module + :param args: Cli args from Arguments() + :param method: Bot running mode + :return: Configuration + """ + config = setup_utils_configuration(args, method) + + no_unlimited_runmodes = { + RunMode.BACKTEST: 'backtesting', + } + if method in no_unlimited_runmodes.keys(): + from freqtrade.data.btanalysis import get_latest_backtest_filename + + btp = Path(config.get('user_data_dir'), "backtest_results") + btfile = get_latest_backtest_filename(btp) + signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" + + if (not os.path.exists(Path(btp, signals_file))): + raise OperationalException( + "Cannot find latest backtest signals file. Run backtesting with --export signals." + ) + + return config + + +def start_analysis_entries_exits(args: Dict[str, Any]) -> None: + """ + Start analysis script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.data.entryexitanalysis import process_entry_exit_reasons + + # Initialize configuration + config = setup_analyze_configuration(args, RunMode.BACKTEST) + + logger.info('Starting freqtrade in analysis mode') + + process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), + config['exchange']['pair_whitelist'], + config['strategy'], + config['analysis_groups'], + config['enter_reason_list'], + config['exit_reason_list'], + config['indicator_list'] + ) From ae1ede58da1193553d5fcb5c85c3911b6e1c8664 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:41:28 +0100 Subject: [PATCH 03/25] Fix import order --- freqtrade/commands/analyze_commands.py | 1 - freqtrade/data/entryexitanalysis.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 0bda9935a..1590dc519 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -1,6 +1,5 @@ import logging import os - from pathlib import Path from typing import Any, Dict diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 62216d5ea..e22a2475e 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,14 +1,13 @@ -import joblib import logging import os - from pathlib import Path from typing import List, Optional +import joblib import pandas as pd from tabulate import tabulate -from freqtrade.data.btanalysis import (load_backtest_data, get_latest_backtest_filename) +from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.exceptions import OperationalException From 80c6190c055f606510abef151886f0607f3fd88a Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 22 May 2022 23:55:59 +0100 Subject: [PATCH 04/25] Fix analyze_commands setup --- freqtrade/commands/analyze_commands.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 1590dc519..a4b3d3f52 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -26,11 +26,10 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btp = Path(config.get('user_data_dir'), "backtest_results") - btfile = get_latest_backtest_filename(btp) + btfile = get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results') signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" - if (not os.path.exists(Path(btp, signals_file))): + if (not os.path.exists(config['user_data_dir'] / 'backtest_results' / signals_file)): raise OperationalException( "Cannot find latest backtest signals file. Run backtesting with --export signals." ) From 8c03ebb78ff637a4545391c2ac62510ba1a1c4f1 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 12:48:13 +0100 Subject: [PATCH 05/25] Fix group 0 table, add pathlib.Path use --- freqtrade/commands/analyze_commands.py | 14 ++++++++++---- freqtrade/commands/cli_options.py | 1 + freqtrade/data/entryexitanalysis.py | 12 +++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index a4b3d3f52..73ae19eaf 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -1,5 +1,4 @@ import logging -import os from pathlib import Path from typing import Any, Dict @@ -26,14 +25,19 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btfile = get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results') - signals_file = f"{os.path.basename(os.path.splitext(btfile)[0])}_signals.pkl" + btfile = Path(get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results')) + signals_file = f"{btfile.stem}_signals.pkl" - if (not os.path.exists(config['user_data_dir'] / 'backtest_results' / signals_file)): + if (not (config['user_data_dir'] / 'backtest_results' / signals_file).exists()): raise OperationalException( "Cannot find latest backtest signals file. Run backtesting with --export signals." ) + if ('strategy' not in config): + raise OperationalException( + "No strategy defined. Use --strategy or supply in config." + ) + return config @@ -48,6 +52,8 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: # Initialize configuration config = setup_analyze_configuration(args, RunMode.BACKTEST) + print(config) + logger.info('Starting freqtrade in analysis mode') process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f925bd699..f76f3688c 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -644,5 +644,6 @@ AVAILABLE_CLI_OPTIONS = { help=("Comma separated list of indicators to analyse. ", "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', + default='', ), } diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index e22a2475e..8bfc940dc 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -97,15 +97,12 @@ def _do_group_table_output(bigdf, glist): 'count', lambda x: sum(x > 0), lambda x: sum(x <= 0)]}) - - new = pd.merge(new, wins, left_index=True, right_index=True) - new = pd.merge(new, loss, left_index=True, right_index=True) + new = pd.concat([new, wins, loss], axis=1).fillna(0) new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) - - new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100) - new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]) - new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]) + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] @@ -249,6 +246,7 @@ def process_entry_exit_reasons(backtest_dir: Path, signal_candles = _load_signal_candles(backtest_dir) analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles) + _print_results(analysed_trades_dict, strategy_name, analysis_groups, From 3adda84b96bdcde1909a6cecb12ef9b3fbd9296c Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 20:27:15 +0100 Subject: [PATCH 06/25] Update docs, add test --- docs/advanced-backtesting.md | 16 ++-- freqtrade/commands/analyze_commands.py | 2 - freqtrade/data/entryexitanalysis.py | 17 ++-- tests/data/test_entryexitanalysis.py | 94 +++++++++++++++++++++++ tests/strategy/strats/strategy_test_v3.py | 9 ++- 5 files changed, 117 insertions(+), 21 deletions(-) create mode 100755 tests/data/test_entryexitanalysis.py diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 2a484da69..4b40bad8e 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -22,23 +22,19 @@ DataFrame of the candles that resulted in buy signals. Depending on how many buy makes, this file may get quite large, so periodically check your `user_data/backtest_results` folder to delete old exports. -To analyze the buy tags, we need to use the `buy_reasons.py` script from -[froggleston's repo](https://github.com/froggleston/freqtrade-buyreasons). Follow the instructions -in their README to copy the script into your `freqtrade/scripts/` folder. - Before running your next backtest, make sure you either delete your old backtest results or run backtesting with the `--cache none` option to make sure no cached results are used. If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -Now run the `buy_reasons.py` script, supplying a few options: +To analyze the entry/exit tags, we now need to use the `freqtrade analysis` command: ``` bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 ``` -The `-g` option is used to specify the various tabular outputs, ranging from the simplest (0) +The `--analysis_groups` option is used to specify the various tabular outputs, ranging from the simplest (0) to the most detailed per pair, per buy and per sell tag (4). More options are available by running with the `-h` option. @@ -54,18 +50,18 @@ To show only certain buy and sell tags in the displayed output, use the followin For example: ```bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" ``` ### Outputting signal candle indicators -The real power of the buy_reasons.py script comes from the ability to print out the indicator +The real power of `freqtrade analysis` comes from the ability to print out the indicator values present on signal candles to allow fine-grained investigation and tuning of buy signal indicators. To print out a column for a given set of indicators, use the `--indicator-list` option: ```bash -python3 scripts/buy_reasons.py -c -s -t -g0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" ``` The indicators have to be present in your strategy's main DataFrame (either for your main diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 73ae19eaf..56330bed3 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -52,8 +52,6 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: # Initialize configuration config = setup_analyze_configuration(args, RunMode.BACKTEST) - print(config) - logger.info('Starting freqtrade in analysis mode') process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 8bfc940dc..9d6c470da 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -15,10 +15,18 @@ logger = logging.getLogger(__name__) def _load_signal_candles(backtest_dir: Path): - scpf = Path(backtest_dir, - os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" - ) + + if backtest_dir.is_dir(): + scpf = Path(backtest_dir, + os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + else: + scpf = Path(os.path.splitext( + get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + ) + + print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -246,7 +254,6 @@ def process_entry_exit_reasons(backtest_dir: Path, signal_candles = _load_signal_candles(backtest_dir) analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles) - _print_results(analysed_trades_dict, strategy_name, analysis_groups, diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py new file mode 100755 index 000000000..548cd88b9 --- /dev/null +++ b/tests/data/test_entryexitanalysis.py @@ -0,0 +1,94 @@ +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pandas as pd + +from freqtrade.commands.analyze_commands import start_analysis_entries_exits +from freqtrade.commands.optimize_commands import start_backtesting +from freqtrade.enums import ExitType +from tests.conftest import get_args, patch_exchange, patched_configuration_load_config_file + + +def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): + default_conf.update({ + "use_exit_signal": True, + "exit_profit_only": False, + "exit_profit_offset": 0.0, + "ignore_roi_if_entry_signal": False, + 'analysis_groups': "0", + 'enter_reason_list': "all", + 'exit_reason_list': "all", + 'indicator_list': "bb_upperband,ema_10" + }) + patch_exchange(mocker) + result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], + 'profit_ratio': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', ], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', ], utc=True), + 'trade_duration': [235, 40], + 'is_open': [False, False], + 'stake_amount': [0.01, 0.01], + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.104969, 0.103541], + "is_short": [False, False], + 'enter_tag': ["enter_tag_long", "enter_tag_long"], + 'exit_reason': [ExitType.ROI, ExitType.ROI] + }) + + backtestmock = MagicMock(side_effect=[ + { + 'results': result1, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'timedout_entry_orders': 0, + 'timedout_exit_orders': 0, + 'canceled_trade_entries': 0, + 'canceled_entry_orders': 0, + 'replaced_entry_orders': 0, + 'final_balance': 1000, + } + ]) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['ETH/BTC', 'LTC/BTC', 'DASH/BTC'])) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'backtesting', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), + '--timeframe', '5m', + '--timerange', '1515560100-1517287800', + '--export', 'signals', + '--cache', 'none', + '--strategy-list', + 'StrategyTestV3', + ] + args = get_args(args) + start_backtesting(args) + + captured = capsys.readouterr() + assert 'BACKTESTING REPORT' in captured.out + assert 'EXIT REASON STATS' in captured.out + assert 'LEFT OPEN TRADES REPORT' in captured.out + + args = [ + 'analysis', + '--config', 'config.json', + '--datadir', str(testdatadir), + '--analysis_groups', '0', + '--strategy', + 'StrategyTestV3', + ] + args = get_args(args) + start_analysis_entries_exits(args) + + captured = capsys.readouterr() + assert 'enter_tag_long' in captured.out diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index df83d3663..f1c9d8e99 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -143,12 +143,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 65) & (dataframe['plus_di'] > self.buy_plusdi.value) ), - 'enter_long'] = 1 + ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - 'enter_short'] = 1 + ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' return dataframe @@ -166,13 +167,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 70) & (dataframe['minus_di'] > self.sell_minusdi.value) ), - 'exit_long'] = 1 + ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' dataframe.loc[ ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - 'exit_short'] = 1 + ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' return dataframe From 22b9805e472f44ce1da2928b6d8177c293012048 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 21:04:23 +0100 Subject: [PATCH 07/25] Fix all tests --- tests/data/test_entryexitanalysis.py | 4 +- tests/rpc/test_rpc_apiserver.py | 6 +- tests/strategy/strats/strategy_test_v3.py | 8 +- .../strats/strategy_test_v3_analysis.py | 195 ++++++++++++++++++ tests/strategy/test_strategy_loading.py | 6 +- 5 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 tests/strategy/strats/strategy_test_v3_analysis.py diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 548cd88b9..151fc3ff8 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -69,7 +69,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--export', 'signals', '--cache', 'none', '--strategy-list', - 'StrategyTestV3', + 'StrategyTestV3Analysis', ] args = get_args(args) start_backtesting(args) @@ -85,7 +85,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--datadir', str(testdatadir), '--analysis_groups', '0', '--strategy', - 'StrategyTestV3', + 'StrategyTestV3Analysis', ] args = get_args(args) start_analysis_entries_exits(args) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 03ba895a1..c887e7776 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1384,12 +1384,16 @@ def test_api_strategies(botclient): rc = client_get(client, f"{BASE_URI}/strategies") assert_response(rc) + + print(rc.json()) + assert rc.json() == {'strategies': [ 'HyperoptableStrategy', 'InformativeDecoratorTest', 'StrategyTestV2', 'StrategyTestV3', - 'StrategyTestV3Futures', + 'StrategyTestV3Analysis', + 'StrategyTestV3Futures' ]} diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index f1c9d8e99..9ca2471bd 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -143,13 +143,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 65) & (dataframe['plus_di'] > self.buy_plusdi.value) ), - ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + 'enter_long'] = 1 dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' + 'enter_short'] = 1 return dataframe @@ -167,13 +167,13 @@ class StrategyTestV3(IStrategy): (dataframe['adx'] > 70) & (dataframe['minus_di'] > self.sell_minusdi.value) ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' + 'exit_long'] = 1 dataframe.loc[ ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' + 'exit_short'] = 1 return dataframe diff --git a/tests/strategy/strats/strategy_test_v3_analysis.py b/tests/strategy/strats/strategy_test_v3_analysis.py new file mode 100644 index 000000000..b237f548f --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_analysis.py @@ -0,0 +1,195 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +from datetime import datetime + +import talib.abstract as ta +from pandas import DataFrame + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.persistence import Trade +from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, + RealParameter) + + +class StrategyTestV3Analysis(IStrategy): + """ + Strategy used by tests freqtrade bot. + Please do not modify this strategy, it's intended for internal use only. + Please look at the SampleStrategy in the user_data/strategy directory + or strategy repository https://github.com/freqtrade/freqtrade-strategies + for samples and inspiration. + """ + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + + # Optional order type mapping + order_types = { + 'entry': 'limit', + 'exit': 'limit', + 'stoploss': 'limit', + 'stoploss_on_exchange': False + } + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + + # Optional time in force for orders + order_time_in_force = { + 'entry': 'gtc', + 'exit': 'gtc', + } + + buy_params = { + 'buy_rsi': 35, + # Intentionally not specified, so "default" is tested + # 'buy_plusdi': 0.4 + } + + sell_params = { + 'sell_rsi': 74, + 'sell_minusdi': 0.4 + } + + buy_rsi = IntParameter([0, 50], default=30, space='buy') + buy_plusdi = RealParameter(low=0, high=1, default=0.5, space='buy') + sell_rsi = IntParameter(low=50, high=100, default=70, space='sell') + sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell', + load=False) + protection_enabled = BooleanParameter(default=True) + protection_cooldown_lookback = IntParameter([0, 50], default=30) + + # TODO: Can this work with protection tests? (replace HyperoptableStrategy implicitly ... ) + # @property + # def protections(self): + # prot = [] + # if self.protection_enabled.value: + # prot.append({ + # "method": "CooldownPeriod", + # "stop_duration_candles": self.protection_cooldown_lookback.value + # }) + # return prot + + bot_started = False + + def bot_start(self): + self.bot_started = True + + def informative_pairs(self): + + return [] + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # Minus Directional Indicator / Movement + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + # EMA - Exponential Moving Average + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + dataframe.loc[ + ( + (dataframe['rsi'] < self.buy_rsi.value) & + (dataframe['fastd'] < 35) & + (dataframe['adx'] > 30) & + (dataframe['plus_di'] > self.buy_plusdi.value) + ) | + ( + (dataframe['adx'] > 65) & + (dataframe['plus_di'] > self.buy_plusdi.value) + ), + ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' + + dataframe.loc[ + ( + qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) + ), + ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + ( + ( + (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) | + (qtpylib.crossed_above(dataframe['fastd'], 70)) + ) & + (dataframe['adx'] > 10) & + (dataframe['minus_di'] > 0) + ) | + ( + (dataframe['adx'] > 70) & + (dataframe['minus_di'] > self.sell_minusdi.value) + ), + ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' + + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) + ), + ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' + + return dataframe + + def leverage(self, pair: str, current_time: datetime, current_rate: float, + proposed_leverage: float, max_leverage: float, side: str, + **kwargs) -> float: + # Return 3.0 in all cases. + # Bot-logic must make sure it's an allowed leverage and eventually adjust accordingly. + + return 3.0 + + def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, min_stake: float, max_stake: float, **kwargs): + + if current_profit < -0.0075: + orders = trade.select_filled_orders(trade.entry_side) + return round(orders[0].cost, 0) + + return None diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 919a4bd00..666ae2b05 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 5 + assert len(strategies) == 6 assert isinstance(strategies[0], dict) @@ -42,10 +42,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 6 + assert len(strategies) == 7 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 5 + assert len([x for x in strategies if x['class'] is not None]) == 6 assert len([x for x in strategies if x['class'] is None]) == 1 From edd474e663b950ade4b4fe172846a8045b013b3e Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 24 May 2022 21:21:20 +0100 Subject: [PATCH 08/25] Another test fix attempt --- .../strats/strategy_test_v3_analysis.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tests/strategy/strats/strategy_test_v3_analysis.py b/tests/strategy/strats/strategy_test_v3_analysis.py index b237f548f..290fef156 100644 --- a/tests/strategy/strats/strategy_test_v3_analysis.py +++ b/tests/strategy/strats/strategy_test_v3_analysis.py @@ -1,12 +1,9 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -from datetime import datetime - import talib.abstract as ta from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.persistence import Trade from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, RealParameter) @@ -176,20 +173,3 @@ class StrategyTestV3Analysis(IStrategy): ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' return dataframe - - def leverage(self, pair: str, current_time: datetime, current_rate: float, - proposed_leverage: float, max_leverage: float, side: str, - **kwargs) -> float: - # Return 3.0 in all cases. - # Bot-logic must make sure it's an allowed leverage and eventually adjust accordingly. - - return 3.0 - - def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, min_stake: float, max_stake: float, **kwargs): - - if current_profit < -0.0075: - orders = trade.select_filled_orders(trade.entry_side) - return round(orders[0].cost, 0) - - return None From 2873ca6d38329245f6aedb84f93f94f1a992eb77 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 09:57:12 +0100 Subject: [PATCH 09/25] Add cleanup, adjust _print_table for indicators, add rsi to test output --- freqtrade/data/entryexitanalysis.py | 7 ++----- tests/data/test_entryexitanalysis.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 9d6c470da..192d666ae 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -26,7 +26,6 @@ def _load_signal_candles(backtest_dir: Path): get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" ) - print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -213,11 +212,9 @@ def _print_results(analysed_trades, stratname, group, if ind in bigdf: available_inds.append(ind) ilist = ["pair", "enter_reason", "exit_reason"] + available_inds - print(tabulate(bigdf[ilist].sort_values(['exit_reason']), - headers='keys', tablefmt='psql', showindex=False)) + _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) else: - print(tabulate(bigdf[columns].sort_values(['pair']), - headers='keys', tablefmt='psql', showindex=False)) + _print_table(bigdf[columns], sortcols=['pair'], show_index=False) else: print("\\_ No trades to show") diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 151fc3ff8..70ba5fa21 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -2,13 +2,22 @@ from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pandas as pd +import pytest from freqtrade.commands.analyze_commands import start_analysis_entries_exits from freqtrade.commands.optimize_commands import start_backtesting from freqtrade.enums import ExitType +from freqtrade.optimize.backtesting import Backtesting from tests.conftest import get_args, patch_exchange, patched_configuration_load_config_file +@pytest.fixture(autouse=True) +def backtesting_cleanup() -> None: + yield None + + Backtesting.cleanup() + + def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): default_conf.update({ "use_exit_signal": True, @@ -18,7 +27,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'analysis_groups': "0", 'enter_reason_list': "all", 'exit_reason_list': "all", - 'indicator_list': "bb_upperband,ema_10" + 'indicator_list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -84,6 +93,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap '--config', 'config.json', '--datadir', str(testdatadir), '--analysis_groups', '0', + '--indicator_list', 'rsi', '--strategy', 'StrategyTestV3Analysis', ] @@ -92,3 +102,6 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out + assert '34.049' in captured.out + + Backtesting.cleanup() From f5c2930889c7d9de7e999817389bff185adb117c Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 09:58:38 +0100 Subject: [PATCH 10/25] Presume that pytest will call the cleanup call --- tests/data/test_entryexitanalysis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 70ba5fa21..3ee986600 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -12,7 +12,7 @@ from tests.conftest import get_args, patch_exchange, patched_configuration_load_ @pytest.fixture(autouse=True) -def backtesting_cleanup() -> None: +def entryexitanalysis_cleanup() -> None: yield None Backtesting.cleanup() @@ -103,5 +103,3 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out assert '34.049' in captured.out - - Backtesting.cleanup() From 21e6c14e1e80e65eff1d50e7d85e0aa330b7983c Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 25 May 2022 10:08:03 +0100 Subject: [PATCH 11/25] Final test changes --- tests/data/test_entryexitanalysis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 3ee986600..9ae89a2f8 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -102,4 +102,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap captured = capsys.readouterr() assert 'enter_tag_long' in captured.out + assert 'ETH/BTC' in captured.out assert '34.049' in captured.out + assert 'LTC/BTC' in captured.out + assert '54.3204' in captured.out From 145faf98178e5b8bc69c7cd1dba3a01eda9d2d7e Mon Sep 17 00:00:00 2001 From: froggleston Date: Thu, 26 May 2022 11:06:38 +0100 Subject: [PATCH 12/25] Use tmpdir for testing --- tests/data/test_entryexitanalysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 9ae89a2f8..ed0bab76b 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -18,7 +18,7 @@ def entryexitanalysis_cleanup() -> None: Backtesting.cleanup() -def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, capsys): +def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmpdir, capsys): default_conf.update({ "use_exit_signal": True, "exit_profit_only": False, @@ -72,6 +72,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'backtesting', '--config', 'config.json', '--datadir', str(testdatadir), + '--user-data-dir', str(tmpdir), '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--timeframe', '5m', '--timerange', '1515560100-1517287800', @@ -92,6 +93,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, cap 'analysis', '--config', 'config.json', '--datadir', str(testdatadir), + '--user-data-dir', str(tmpdir), '--analysis_groups', '0', '--indicator_list', 'rsi', '--strategy', From 43b7955fc2b43a78102f802ed5dee9e348564823 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 19:37:55 +0200 Subject: [PATCH 13/25] Fully rely on pathlib --- freqtrade/data/entryexitanalysis.py | 10 +++------- tests/rpc/test_rpc_apiserver.py | 2 -- tests/strategy/strats/strategy_test_v3.py | 1 - 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 192d666ae..3c83d4abf 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,5 +1,4 @@ import logging -import os from pathlib import Path from typing import List, Optional @@ -18,14 +17,11 @@ def _load_signal_candles(backtest_dir: Path): if backtest_dir.is_dir(): scpf = Path(backtest_dir, - os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" + Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: - scpf = Path(os.path.splitext( - get_latest_backtest_filename(backtest_dir))[0] + "_signals.pkl" - ) - + scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") + print(scpf) try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index c887e7776..8b3ac18ac 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1385,8 +1385,6 @@ def test_api_strategies(botclient): assert_response(rc) - print(rc.json()) - assert rc.json() == {'strategies': [ 'HyperoptableStrategy', 'InformativeDecoratorTest', diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 9ca2471bd..df83d3663 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -144,7 +144,6 @@ class StrategyTestV3(IStrategy): (dataframe['plus_di'] > self.buy_plusdi.value) ), 'enter_long'] = 1 - dataframe.loc[ ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) From e7c5818d1645af2e373f346cee98e3a69e47395b Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 11:20:11 +0100 Subject: [PATCH 14/25] First pass changes for cleaning up --- freqtrade/commands/arguments.py | 6 +- freqtrade/commands/cli_options.py | 8 +- freqtrade/data/entryexitanalysis.py | 145 ++++++++++++--------------- tests/data/test_entryexitanalysis.py | 12 +-- 4 files changed, 75 insertions(+), 96 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 4dd0141fa..679193e49 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,8 +101,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", - "exit_reason_list", "indicator_list"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis-groups", "enter-reason-list", + "exit-reason-list", "indicator-list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", @@ -421,7 +421,7 @@ class Arguments: self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('analysis', help='Analysis module.', + analysis_cmd = subparsers.add_parser('analysis', help='Backtest Analysis module.', parents=[_common_parser, _strategy_parser]) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index f76f3688c..ce7320b95 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -615,7 +615,7 @@ AVAILABLE_CLI_OPTIONS = { action="store_true", ), "analysis_groups": Arg( - "--analysis_groups", + "--analysis-groups", help=("grouping output - ", "0: simple wins/losses by enter tag, ", "1: by enter_tag, ", @@ -626,21 +626,21 @@ AVAILABLE_CLI_OPTIONS = { default="0,1,2", ), "enter_reason_list": Arg( - "--enter_reason_list", + "--enter-reason-list", help=("Comma separated list of entry signals to analyse. Default: all. ", "e.g. 'entry_tag_a,entry_tag_b'"), nargs='?', default='all', ), "exit_reason_list": Arg( - "--exit_reason_list", + "--exit-reason-list", help=("Comma separated list of exit signals to analyse. Default: all. ", "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), nargs='?', default='all', ), "indicator_list": Arg( - "--indicator_list", + "--indicator-list", help=("Comma separated list of indicators to analyse. ", "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 192d666ae..53a256633 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -7,7 +7,8 @@ import joblib import pandas as pd from tabulate import tabulate -from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data +from freqtrade.data.btanalysis import (get_latest_backtest_filename, load_backtest_data, + load_backtest_stats) from freqtrade.exceptions import OperationalException @@ -49,8 +50,8 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand pair, trades, signal_candles[strategy_name][pair]) - except Exception: - pass + except Exception as e: + print(f"Cannot process entry/exit reasons for {strategy_name}: ", e) return analysed_trades_dict @@ -82,104 +83,79 @@ def _analyze_candles_and_indicators(pair, trades, signal_candles): try: trades_red = pd.merge(trades_red, trades_inds, on='signal_date', how='outer') except Exception as e: - print(e) + raise e return trades_red else: return pd.DataFrame() def _do_group_table_output(bigdf, glist): - if "0" in glist: - wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ - .groupby(['enter_reason']) \ - .agg({'profit_abs': ['sum']}) + for g in glist: + # 0: summary wins/losses grouped by enter tag + if g == "0": + group_mask = ['enter_reason'] + wins = bigdf.loc[bigdf['profit_abs'] >= 0] \ + .groupby(group_mask) \ + .agg({'profit_abs': ['sum']}) - wins.columns = ['profit_abs_wins'] - loss = bigdf.loc[bigdf['profit_abs'] < 0] \ - .groupby(['enter_reason']) \ - .agg({'profit_abs': ['sum']}) - loss.columns = ['profit_abs_loss'] + wins.columns = ['profit_abs_wins'] + loss = bigdf.loc[bigdf['profit_abs'] < 0] \ + .groupby(group_mask) \ + .agg({'profit_abs': ['sum']}) + loss.columns = ['profit_abs_loss'] - new = bigdf.groupby(['enter_reason']).agg({'profit_abs': [ - 'count', - lambda x: sum(x > 0), - lambda x: sum(x <= 0)]}) - new = pd.concat([new, wins, loss], axis=1).fillna(0) + new = bigdf.groupby(group_mask).agg({'profit_abs': [ + 'count', + lambda x: sum(x > 0), + lambda x: sum(x <= 0)]}) + new = pd.concat([new, wins, loss], axis=1).fillna(0) - new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) - new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) - new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) - new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) + new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss']) + new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0) + new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) + new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) - new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', - 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] + new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', + 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] - sortcols = ['total_num_buys'] + sortcols = ['total_num_buys'] - _print_table(new, sortcols, show_index=True) - if "1" in glist: - new = bigdf.groupby(['enter_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['enter_reason', 'num_buys', 'profit_abs_sum', 'profit_abs_median', - 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', - 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + _print_table(new, sortcols, show_index=True) - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) - if "2" in glist: - new = bigdf.groupby(['enter_reason', 'exit_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] - - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) - if "3" in glist: - new = bigdf.groupby(['pair', 'enter_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], + else: + agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'], 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['pair', 'enter_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median', + 'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct', + 'total_profit_pct'] + sortcols = ['profit_abs_sum', 'enter_reason'] - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 + # 1: profit summaries grouped by enter_tag + if g == "1": + group_mask = ['enter_reason'] - _print_table(new, sortcols) - if "4" in glist: - new = bigdf.groupby(['pair', 'enter_reason', 'exit_reason']) \ - .agg({'profit_abs': ['count', 'sum', 'median', 'mean'], - 'profit_ratio': ['sum', 'median', 'mean']} - ).reset_index() - new.columns = ['pair', 'enter_reason', 'exit_reason', 'num_buys', 'profit_abs_sum', - 'profit_abs_median', 'profit_abs_mean', 'median_profit_pct', - 'mean_profit_pct', 'total_profit_pct'] - sortcols = ['profit_abs_sum', 'enter_reason'] + # 2: profit summaries grouped by enter_tag and exit_tag + if g == "2": + group_mask = ['enter_reason', 'exit_reason'] - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 + # 3: profit summaries grouped by pair and enter_tag + if g == "3": + group_mask = ['pair', 'enter_reason'] - _print_table(new, sortcols) + # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) + if g == "4": + group_mask = ['pair', 'enter_reason', 'exit_reason'] + + new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() + new.columns = group_mask + agg_cols + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 + + _print_table(new, sortcols) -def _print_results(analysed_trades, stratname, group, +def _print_results(analysed_trades, stratname, analysis_groups, enter_reason_list, exit_reason_list, indicator_list, columns=None): @@ -191,8 +167,8 @@ def _print_results(analysed_trades, stratname, group, bigdf = pd.concat([bigdf, trades], ignore_index=True) if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): - if group is not None: - glist = group.split(",") + if analysis_groups is not None: + glist = analysis_groups.split(",") _do_group_table_output(bigdf, glist) if enter_reason_list is not None and not enter_reason_list == "all": @@ -244,6 +220,9 @@ def process_entry_exit_reasons(backtest_dir: Path, indicator_list: Optional[str] = None): try: + bt_stats = load_backtest_stats(backtest_dir) + logger.info(bt_stats) + # strategy_name = bt_stats['something'] trades = load_backtest_data(backtest_dir, strategy_name) except ValueError as e: raise OperationalException(e) from e diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index ed0bab76b..90da80ce9 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -24,10 +24,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "exit_profit_only": False, "exit_profit_offset": 0.0, "ignore_roi_if_entry_signal": False, - 'analysis_groups': "0", - 'enter_reason_list': "all", - 'exit_reason_list': "all", - 'indicator_list': "rsi" + 'analysis-groups': "0", + 'enter-reason-list': "all", + 'exit-reason-list': "all", + 'indicator-list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -94,8 +94,8 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--analysis_groups', '0', - '--indicator_list', 'rsi', + '--analysis-groups', '0', + '--indicator-list', 'rsi', '--strategy', 'StrategyTestV3Analysis', ] From df1c36e5aa7085a78ee06ab2cdd1558b0ccd7331 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 11:54:27 +0100 Subject: [PATCH 15/25] Change command name, use load_backtest_stats for strategy resolving --- freqtrade/commands/analyze_commands.py | 1 - freqtrade/commands/arguments.py | 7 +++--- freqtrade/data/entryexitanalysis.py | 35 ++++++++++++-------------- tests/data/test_entryexitanalysis.py | 13 ++++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 56330bed3..2fa13f683 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -56,7 +56,6 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), config['exchange']['pair_whitelist'], - config['strategy'], config['analysis_groups'], config['enter_reason_list'], config['exit_reason_list'], diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 679193e49..d5831a2ac 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,8 +101,8 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis-groups", "enter-reason-list", - "exit-reason-list", "indicator-list"] +ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", + "exit_reason_list", "indicator_list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", @@ -421,7 +421,8 @@ class Arguments: self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('analysis', help='Backtest Analysis module.', + analysis_cmd = subparsers.add_parser('backtesting-analysis', + help='Backtest Analysis module.', parents=[_common_parser, _strategy_parser]) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 1ee6eea42..15ac6ba09 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -15,14 +15,13 @@ logger = logging.getLogger(__name__) def _load_signal_candles(backtest_dir: Path): - if backtest_dir.is_dir(): scpf = Path(backtest_dir, Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") - print(scpf) + try: scp = open(scpf, "rb") signal_candles = joblib.load(scp) @@ -154,7 +153,6 @@ def _do_group_table_output(bigdf, glist): def _print_results(analysed_trades, stratname, analysis_groups, enter_reason_list, exit_reason_list, indicator_list, columns=None): - if columns is None: columns = ['pair', 'open_date', 'close_date', 'profit_abs', 'enter_reason', 'exit_reason'] @@ -209,26 +207,25 @@ def _print_table(df, sortcols=None, show_index=False): def process_entry_exit_reasons(backtest_dir: Path, pairlist: List[str], - strategy_name: str, analysis_groups: Optional[str] = "0,1,2", enter_reason_list: Optional[str] = "all", exit_reason_list: Optional[str] = "all", indicator_list: Optional[str] = None): - try: - bt_stats = load_backtest_stats(backtest_dir) - logger.info(bt_stats) - # strategy_name = bt_stats['something'] - trades = load_backtest_data(backtest_dir, strategy_name) + backtest_stats = load_backtest_stats(backtest_dir) + for strategy_name, results in backtest_stats['strategy'].items(): + trades = load_backtest_data(backtest_dir, strategy_name) + + if not trades.empty: + signal_candles = _load_signal_candles(backtest_dir) + analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, + trades, signal_candles) + _print_results(analysed_trades_dict, + strategy_name, + analysis_groups, + enter_reason_list, + exit_reason_list, + indicator_list) + except ValueError as e: raise OperationalException(e) from e - if not trades.empty: - signal_candles = _load_signal_candles(backtest_dir) - analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name, - trades, signal_candles) - _print_results(analysed_trades_dict, - strategy_name, - analysis_groups, - enter_reason_list, - exit_reason_list, - indicator_list) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 90da80ce9..eadf79179 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -24,10 +24,6 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "exit_profit_only": False, "exit_profit_offset": 0.0, "ignore_roi_if_entry_signal": False, - 'analysis-groups': "0", - 'enter-reason-list': "all", - 'exit-reason-list': "all", - 'indicator-list': "rsi" }) patch_exchange(mocker) result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], @@ -89,8 +85,15 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert 'EXIT REASON STATS' in captured.out assert 'LEFT OPEN TRADES REPORT' in captured.out + default_conf.update({ + 'analysis_groups': "0", + 'enter_reason_list': "all", + 'exit_reason_list': "all", + 'indicator_list': "rsi" + }) + args = [ - 'analysis', + 'backtesting-analysis', '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), From 24b02127ec03e7c7e05b13fb450310980a7663a4 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 15:42:34 +0100 Subject: [PATCH 16/25] Update docs --- docs/advanced-backtesting.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 4b40bad8e..7f2be1f1a 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -28,40 +28,47 @@ backtesting with the `--cache none` option to make sure no cached results are us If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -To analyze the entry/exit tags, we now need to use the `freqtrade analysis` command: +To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command: ``` bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 ``` -The `--analysis_groups` option is used to specify the various tabular outputs, ranging from the simplest (0) -to the most detailed per pair, per buy and per sell tag (4). More options are available by -running with the `-h` option. +This command will read from the last backtesting results. The `--analysis-groups` option is +used to specify the various tabular outputs showing the profit fo each group or trade, +ranging from the simplest (0) to the most detailed per pair, per buy and per sell tag (4): + +* 1: profit summaries grouped by enter_tag +* 2: profit summaries grouped by enter_tag and exit_tag +* 3: profit summaries grouped by pair and enter_tag +* 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) + +More options are available by running with the `-h` option. ### Tuning the buy tags and sell tags to display To show only certain buy and sell tags in the displayed output, use the following two options: ``` ---enter_reason_list : Comma separated list of enter signals to analyse. Default: "all" ---exit_reason_list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" +--enter-reason-list : Comma separated list of enter signals to analyse. Default: "all" +--exit-reason-list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" ``` For example: ```bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" ``` ### Outputting signal candle indicators -The real power of `freqtrade analysis` comes from the ability to print out the indicator +The real power of `freqtrade backtesting-analysis` comes from the ability to print out the indicator values present on signal candles to allow fine-grained investigation and tuning of buy signal indicators. To print out a column for a given set of indicators, use the `--indicator-list` option: ```bash -freqtrade analysis -c -s --analysis_groups 0,1,2,3,4 --enter_reason_list "enter_tag_a,enter_tag_b" --exit_reason_list "roi,custom_exit_tag_a,stop_loss" --indicator_list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" --indicator-list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" ``` The indicators have to be present in your strategy's main DataFrame (either for your main From 9a068c0b14ebb80df49d917223469e950ba4a358 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 29 May 2022 16:25:31 +0100 Subject: [PATCH 17/25] Add test for each analysis group, remove default table output if not indicator-list --- freqtrade/data/entryexitanalysis.py | 4 +- tests/data/test_entryexitanalysis.py | 149 +++++++++++++++++++++------ 2 files changed, 119 insertions(+), 34 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 15ac6ba09..1c21fcc15 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -173,7 +173,7 @@ def _print_results(analysed_trades, stratname, analysis_groups, exit_reason_list = exit_reason_list.split(",") bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] - if indicator_list is not None: + if indicator_list is not None and indicator_list != "": if indicator_list == "all": print(bigdf) else: @@ -183,8 +183,6 @@ def _print_results(analysed_trades, stratname, analysis_groups, available_inds.append(ind) ilist = ["pair", "enter_reason", "exit_reason"] + available_inds _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) - else: - _print_table(bigdf[columns], sortcols=['pair'], show_index=False) else: print("\\_ No trades to show") diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index eadf79179..971cb51aa 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -19,6 +20,8 @@ def entryexitanalysis_cleanup() -> None: def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmpdir, capsys): + caplog.set_level(logging.INFO) + default_conf.update({ "use_exit_signal": True, "exit_profit_only": False, @@ -26,22 +29,32 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp "ignore_roi_if_entry_signal": False, }) patch_exchange(mocker) - result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC'], - 'profit_ratio': [0.0, 0.0], - 'profit_abs': [0.0, 0.0], + result1 = pd.DataFrame({'pair': ['ETH/BTC', 'LTC/BTC', 'ETH/BTC', 'LTC/BTC'], + 'profit_ratio': [0.025, 0.05, -0.1, -0.05], + 'profit_abs': [0.5, 2.0, -4.0, -2.0], 'open_date': pd.to_datetime(['2018-01-29 18:40:00', - '2018-01-30 03:30:00', ], utc=True + '2018-01-30 03:30:00', + '2018-01-30 08:10:00', + '2018-01-31 13:30:00', ], utc=True ), 'close_date': pd.to_datetime(['2018-01-29 20:45:00', - '2018-01-30 05:35:00', ], utc=True), - 'trade_duration': [235, 40], - 'is_open': [False, False], - 'stake_amount': [0.01, 0.01], - 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.104969, 0.103541], - "is_short": [False, False], - 'enter_tag': ["enter_tag_long", "enter_tag_long"], - 'exit_reason': [ExitType.ROI, ExitType.ROI] + '2018-01-30 05:35:00', + '2018-01-30 09:10:00', + '2018-01-31 15:00:00', ], utc=True), + 'trade_duration': [235, 40, 60, 90], + 'is_open': [False, False, False, False], + 'stake_amount': [0.01, 0.01, 0.01, 0.01], + 'open_rate': [0.104445, 0.10302485, 0.10302485, 0.10302485], + 'close_rate': [0.104969, 0.103541, 0.102041, 0.102541], + "is_short": [False, False, False, False], + 'enter_tag': ["enter_tag_long_a", + "enter_tag_long_b", + "enter_tag_long_a", + "enter_tag_long_b"], + 'exit_reason': [ExitType.ROI, + ExitType.EXIT_SIGNAL, + ExitType.STOP_LOSS, + ExitType.TRAILING_STOP_LOSS] }) backtestmock = MagicMock(side_effect=[ @@ -85,29 +98,103 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert 'EXIT REASON STATS' in captured.out assert 'LEFT OPEN TRADES REPORT' in captured.out - default_conf.update({ - 'analysis_groups': "0", - 'enter_reason_list': "all", - 'exit_reason_list': "all", - 'indicator_list': "rsi" - }) - - args = [ + base_args = [ 'backtesting-analysis', '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--analysis-groups', '0', - '--indicator-list', 'rsi', - '--strategy', - 'StrategyTestV3Analysis', ] - args = get_args(args) - start_analysis_entries_exits(args) + strat_args = ['--strategy', 'StrategyTestV3Analysis'] + # test group 0 and indicator list + args = get_args(base_args + + ['--analysis-groups', '0', + '--indicator-list', 'close,rsi,profit_abs'] + + strat_args) + start_analysis_entries_exits(args) captured = capsys.readouterr() - assert 'enter_tag_long' in captured.out - assert 'ETH/BTC' in captured.out - assert '34.049' in captured.out assert 'LTC/BTC' in captured.out - assert '54.3204' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert '0.5' in captured.out + assert '-4' in captured.out + assert '-2' in captured.out + assert '-3.5' in captured.out + assert '50' in captured.out + assert '0' in captured.out + assert '0.01616' in captured.out + assert '34.049' in captured.out + assert '0.104104' in captured.out + assert '47.0996' in captured.out + + # test group 1 + args = get_args(base_args + ['--analysis-groups', '1'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'total_profit_pct' in captured.out + assert '-3.5' in captured.out + assert '-1.75' in captured.out + assert '-7.5' in captured.out + assert '-3.75' in captured.out + assert '0' in captured.out + + # test group 2 + args = get_args(base_args + ['--analysis-groups', '2'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert 'total_profit_pct' in captured.out + assert '-10' in captured.out + assert '-5' in captured.out + assert '2.5' in captured.out + + # test group 3 + args = get_args(base_args + ['--analysis-groups', '3'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'LTC/BTC' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'total_profit_pct' in captured.out + assert '-7.5' in captured.out + assert '-3.75' in captured.out + assert '-1.75' in captured.out + assert '0' in captured.out + assert '2' in captured.out + + # test group 4 + args = get_args(base_args + ['--analysis-groups', '4'] + + strat_args) + start_analysis_entries_exits(args) + captured = capsys.readouterr() + assert 'LTC/BTC' in captured.out + assert 'ETH/BTC' in captured.out + assert 'enter_tag_long_a' in captured.out + assert 'enter_tag_long_b' in captured.out + assert 'exit_signal' in captured.out + assert 'roi' in captured.out + assert 'stop_loss' in captured.out + assert 'trailing_stop_loss' in captured.out + assert 'total_profit_pct' in captured.out + assert '-10' in captured.out + assert '-5' in captured.out + assert '-4' in captured.out + assert '0.5' in captured.out + assert '1' in captured.out + assert '2.5' in captured.out From 056047f6352dcbe473890b8ec0df258ea55abdea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 May 2022 20:07:02 +0200 Subject: [PATCH 18/25] Fix --help --- freqtrade/commands/cli_options.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index ce7320b95..e90d3478d 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -616,32 +616,32 @@ AVAILABLE_CLI_OPTIONS = { ), "analysis_groups": Arg( "--analysis-groups", - help=("grouping output - ", - "0: simple wins/losses by enter tag, ", - "1: by enter_tag, ", - "2: by enter_tag and exit_tag, ", - "3: by pair and enter_tag, ", + help=("grouping output - " + "0: simple wins/losses by enter tag, " + "1: by enter_tag, " + "2: by enter_tag and exit_tag, " + "3: by pair and enter_tag, " "4: by pair, enter_ and exit_tag (this can get quite large)"), nargs='?', default="0,1,2", ), "enter_reason_list": Arg( "--enter-reason-list", - help=("Comma separated list of entry signals to analyse. Default: all. ", + help=("Comma separated list of entry signals to analyse. Default: all. " "e.g. 'entry_tag_a,entry_tag_b'"), nargs='?', default='all', ), "exit_reason_list": Arg( "--exit-reason-list", - help=("Comma separated list of exit signals to analyse. Default: all. ", + help=("Comma separated list of exit signals to analyse. Default: all. " "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), nargs='?', default='all', ), "indicator_list": Arg( "--indicator-list", - help=("Comma separated list of indicators to analyse. ", + help=("Comma separated list of indicators to analyse. " "e.g. 'close,rsi,bb_lowerband,profit_abs'"), nargs='?', default='', From c285ad0e2bbd39d2c5e59a38034392b23fb05491 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 May 2022 20:26:24 +0200 Subject: [PATCH 19/25] Remove --strategy parameters, update docs --- docs/utils.md | 50 +++++++++++++++++++++++++++++++++ freqtrade/commands/arguments.py | 15 +++++----- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 9b799e5fc..f87aa2ffc 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -651,6 +651,56 @@ Common arguments: ``` +## Detailed backtest analysis + +Advanced backtest result analysis. + +More details in the [Backtesting analysis](advanced-backtesting.md#analyze-the-buyentry-and-sellexit-tags) Section. + +``` +usage: freqtrade backtesting-analysis [-h] [-v] [--logfile FILE] [-V] + [-c PATH] [-d PATH] [--userdir PATH] + [--analysis-groups [ANALYSIS_GROUPS]] + [--enter-reason-list [ENTER_REASON_LIST]] + [--exit-reason-list [EXIT_REASON_LIST]] + [--indicator-list [INDICATOR_LIST]] + +optional arguments: + -h, --help show this help message and exit + --analysis-groups [ANALYSIS_GROUPS] + grouping output - 0: simple wins/losses by enter tag, + 1: by enter_tag, 2: by enter_tag and exit_tag, 3: by + pair and enter_tag, 4: by pair, enter_ and exit_tag + (this can get quite large) + --enter-reason-list [ENTER_REASON_LIST] + Comma separated list of entry signals to analyse. + Default: all. e.g. 'entry_tag_a,entry_tag_b' + --exit-reason-list [EXIT_REASON_LIST] + Comma separated list of exit signals to analyse. + Default: all. e.g. + 'exit_tag_a,roi,stop_loss,trailing_stop_loss' + --indicator-list [INDICATOR_LIST] + Comma separated list of indicators to analyse. e.g. + 'close,rsi,bb_lowerband,profit_abs' + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + ## List Hyperopt results You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command. diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index d5831a2ac..aed96d042 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -287,6 +287,14 @@ class Arguments: backtesting_show_cmd.set_defaults(func=start_backtesting_show) self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd) + # Add backtesting analysis subcommand + analysis_cmd = subparsers.add_parser('backtesting-analysis', + help='Backtest Analysis module.', + parents=[_common_parser]) + analysis_cmd.set_defaults(func=start_analysis_entries_exits) + self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) + + # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.', parents=[_common_parser, _strategy_parser]) @@ -419,10 +427,3 @@ class Arguments: parents=[_common_parser]) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) - - # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('backtesting-analysis', - help='Backtest Analysis module.', - parents=[_common_parser, _strategy_parser]) - analysis_cmd.set_defaults(func=start_analysis_entries_exits) - self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) From be6e0813db1e3bc45f33381c67b8129cd3404de4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 May 2022 06:34:08 +0200 Subject: [PATCH 20/25] Remove --strategy from analysis test --- freqtrade/commands/arguments.py | 1 - tests/data/test_entryexitanalysis.py | 21 ++++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index aed96d042..6092c630b 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -294,7 +294,6 @@ class Arguments: analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) - # Add edge subcommand edge_cmd = subparsers.add_parser('edge', help='Edge module.', parents=[_common_parser, _strategy_parser]) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 971cb51aa..6209110fe 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -1,5 +1,4 @@ import logging -from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pandas as pd @@ -82,13 +81,10 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--config', 'config.json', '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), - '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--timeframe', '5m', '--timerange', '1515560100-1517287800', '--export', 'signals', '--cache', 'none', - '--strategy-list', - 'StrategyTestV3Analysis', ] args = get_args(args) start_backtesting(args) @@ -104,13 +100,12 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp '--datadir', str(testdatadir), '--user-data-dir', str(tmpdir), ] - strat_args = ['--strategy', 'StrategyTestV3Analysis'] # test group 0 and indicator list args = get_args(base_args + ['--analysis-groups', '0', - '--indicator-list', 'close,rsi,profit_abs'] + - strat_args) + '--indicator-list', 'close,rsi,profit_abs'] + ) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -133,8 +128,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '47.0996' in captured.out # test group 1 - args = get_args(base_args + ['--analysis-groups', '1'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '1']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -147,8 +141,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '0' in captured.out # test group 2 - args = get_args(base_args + ['--analysis-groups', '2'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '2']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -163,8 +156,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2.5' in captured.out # test group 3 - args = get_args(base_args + ['--analysis-groups', '3'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '3']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -179,8 +171,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2' in captured.out # test group 4 - args = get_args(base_args + ['--analysis-groups', '4'] + - strat_args) + args = get_args(base_args + ['--analysis-groups', '4']) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out From 6bb342f23a28559eca7c2355edd6e4af73b7e112 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 14 Jun 2022 16:54:27 +0100 Subject: [PATCH 21/25] Add export-filename support --- docs/advanced-backtesting.md | 25 +++++++++++++++++++++++++ freqtrade/commands/analyze_commands.py | 26 ++++++++++++++++---------- freqtrade/commands/arguments.py | 2 +- freqtrade/data/entryexitanalysis.py | 2 +- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 7f2be1f1a..457c487e9 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -45,6 +45,31 @@ ranging from the simplest (0) to the most detailed per pair, per buy and per sel More options are available by running with the `-h` option. +### Using export-filename + +Normally, `backtesting-analysis` uses the latest backtest results, but if you wanted to go +back to a previous backtest output, you need to supply the `--export-filename` option. +You can supply the same parameter to `backtest-analysis` with the name of the final backtest +output file. This allows you to keep historical versions of backtest results and reanalyse +them at a later date: + +``` bash +freqtrade backtesting -c --timeframe --strategy --timerange= --export=signals --export-filename=/tmp/mystrat_backtest.json +``` + +You should see some output similar to below in the logs with the name of the timestamped +filename that was exported: + +``` +2022-06-14 16:28:32,698 - freqtrade.misc - INFO - dumping json to "/tmp/mystrat_backtest-2022-06-14_16-28-32.json" +``` + +You can then use that filename in `backtesting-analysis`: + +``` +freqtrade backtesting-analysis -c --export-filename=/tmp/mystrat_backtest-2022-06-14_16-28-32.json +``` + ### Tuning the buy tags and sell tags to display To show only certain buy and sell tags in the displayed output, use the following two options: diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 2fa13f683..b6b790788 100755 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -25,17 +25,23 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s if method in no_unlimited_runmodes.keys(): from freqtrade.data.btanalysis import get_latest_backtest_filename - btfile = Path(get_latest_backtest_filename(config['user_data_dir'] / 'backtest_results')) - signals_file = f"{btfile.stem}_signals.pkl" + if 'exportfilename' in config: + if config['exportfilename'].is_dir(): + btfile = Path(get_latest_backtest_filename(config['exportfilename'])) + signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl" + else: + if config['exportfilename'].exists(): + btfile = Path(config['exportfilename']) + signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl" + else: + raise OperationalException(f"{config['exportfilename']} does not exist.") + else: + raise OperationalException('exportfilename not in config.') - if (not (config['user_data_dir'] / 'backtest_results' / signals_file).exists()): + if (not Path(signals_file).exists()): raise OperationalException( - "Cannot find latest backtest signals file. Run backtesting with --export signals." - ) - - if ('strategy' not in config): - raise OperationalException( - "No strategy defined. Use --strategy or supply in config." + (f"Cannot find latest backtest signals file: {signals_file}." + "Run backtesting with `--export signals`.") ) return config @@ -54,7 +60,7 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None: logger.info('Starting freqtrade in analysis mode') - process_entry_exit_reasons(Path(config['user_data_dir'], 'backtest_results'), + process_entry_exit_reasons(config['exportfilename'], config['exchange']['pair_whitelist'], config['analysis_groups'], config['enter_reason_list'], diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 6092c630b..1e3e2845a 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -101,7 +101,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", "disableparamexport", "backtest_breakdown"] -ARGS_ANALYZE_ENTRIES_EXITS = ["analysis_groups", "enter_reason_list", +ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason_list", "exit_reason_list", "indicator_list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 1c21fcc15..d67064bd7 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -20,7 +20,7 @@ def _load_signal_candles(backtest_dir: Path): Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl" ) else: - scpf = Path(Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl") + scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_signals.pkl") try: scp = open(scpf, "rb") From 29d8aeb9b3c19c5b5c7a37d8d8dc42e6478f6f33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 07:13:47 +0200 Subject: [PATCH 22/25] Don't fail on invalid parameter --- freqtrade/data/entryexitanalysis.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index d67064bd7..999f27955 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -140,14 +140,16 @@ def _do_group_table_output(bigdf, glist): # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large) if g == "4": group_mask = ['pair', 'enter_reason', 'exit_reason'] + if group_mask: + new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() + new.columns = group_mask + agg_cols + new['median_profit_pct'] = new['median_profit_pct'] * 100 + new['mean_profit_pct'] = new['mean_profit_pct'] * 100 + new['total_profit_pct'] = new['total_profit_pct'] * 100 - new = bigdf.groupby(group_mask).agg(agg_mask).reset_index() - new.columns = group_mask + agg_cols - new['median_profit_pct'] = new['median_profit_pct'] * 100 - new['mean_profit_pct'] = new['mean_profit_pct'] * 100 - new['total_profit_pct'] = new['total_profit_pct'] * 100 - - _print_table(new, sortcols) + _print_table(new, sortcols) + else: + logger.warning("Invalid group mask specified.") def _print_results(analysed_trades, stratname, analysis_groups, From c391ca08ded88ff1c4edfd8ab9a40b385b0a16a2 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 15 Jun 2022 11:25:06 +0100 Subject: [PATCH 23/25] Change backtesting-analysis options to space separated lists --- docs/advanced-backtesting.md | 13 +++++----- freqtrade/commands/cli_options.py | 17 +++++++------ freqtrade/data/entryexitanalysis.py | 38 +++++++++++++---------------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index 457c487e9..b6b75c47d 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -28,10 +28,11 @@ backtesting with the `--cache none` option to make sure no cached results are us If all goes well, you should now see a `backtest-result-{timestamp}_signals.pkl` file in the `user_data/backtest_results` folder. -To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command: +To analyze the entry/exit tags, we now need to use the `freqtrade backtesting-analysis` command +with `--analysis-groups` option provided with space-separated arguments (default `0 1 2`): ``` bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 +freqtrade backtesting-analysis -c --analysis-groups 0 1 2 3 4 ``` This command will read from the last backtesting results. The `--analysis-groups` option is @@ -75,14 +76,14 @@ freqtrade backtesting-analysis -c --export-filename=/tmp/mystrat_b To show only certain buy and sell tags in the displayed output, use the following two options: ``` ---enter-reason-list : Comma separated list of enter signals to analyse. Default: "all" ---exit-reason-list : Comma separated list of exit signals to analyse. Default: "stop_loss,trailing_stop_loss" +--enter-reason-list : Space-separated list of enter signals to analyse. Default: "all" +--exit-reason-list : Space-separated list of exit signals to analyse. Default: "all" ``` For example: ```bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" +freqtrade backtesting-analysis -c --analysis-groups 0 2 --enter-reason-list enter_tag_a enter_tag_b --exit-reason-list roi custom_exit_tag_a stop_loss ``` ### Outputting signal candle indicators @@ -93,7 +94,7 @@ indicators. To print out a column for a given set of indicators, use the `--indi option: ```bash -freqtrade backtesting-analysis -c --analysis-groups 0,1,2,3,4 --enter-reason-list "enter_tag_a,enter_tag_b" --exit-reason-list "roi,custom_exit_tag_a,stop_loss" --indicator-list "rsi,rsi_1h,bb_lowerband,ema_9,macd,macdsignal" +freqtrade backtesting-analysis -c --analysis-groups 0 2 --enter-reason-list enter_tag_a enter_tag_b --exit-reason-list roi custom_exit_tag_a stop_loss --indicator-list rsi rsi_1h bb_lowerband ema_9 macd macdsignal ``` The indicators have to be present in your strategy's main DataFrame (either for your main diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index e90d3478d..3370ce64b 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -622,28 +622,29 @@ AVAILABLE_CLI_OPTIONS = { "2: by enter_tag and exit_tag, " "3: by pair and enter_tag, " "4: by pair, enter_ and exit_tag (this can get quite large)"), - nargs='?', - default="0,1,2", + nargs='+', + default=['0', '1', '2'], + choices=['0', '1', '2', '3', '4'], ), "enter_reason_list": Arg( "--enter-reason-list", help=("Comma separated list of entry signals to analyse. Default: all. " "e.g. 'entry_tag_a,entry_tag_b'"), - nargs='?', - default='all', + nargs='+', + default=['all'], ), "exit_reason_list": Arg( "--exit-reason-list", help=("Comma separated list of exit signals to analyse. Default: all. " "e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"), - nargs='?', - default='all', + nargs='+', + default=['all'], ), "indicator_list": Arg( "--indicator-list", help=("Comma separated list of indicators to analyse. " "e.g. 'close,rsi,bb_lowerband,profit_abs'"), - nargs='?', - default='', + nargs='+', + default=[], ), } diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index d67064bd7..6a157debb 100755 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -161,28 +161,24 @@ def _print_results(analysed_trades, stratname, analysis_groups, bigdf = pd.concat([bigdf, trades], ignore_index=True) if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns): - if analysis_groups is not None: - glist = analysis_groups.split(",") - _do_group_table_output(bigdf, glist) + if analysis_groups: + _do_group_table_output(bigdf, analysis_groups) - if enter_reason_list is not None and not enter_reason_list == "all": - enter_reason_list = enter_reason_list.split(",") + if enter_reason_list and "all" not in enter_reason_list: bigdf = bigdf.loc[(bigdf['enter_reason'].isin(enter_reason_list))] - if exit_reason_list is not None and not exit_reason_list == "all": - exit_reason_list = exit_reason_list.split(",") + if exit_reason_list and "all" not in exit_reason_list: bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))] - if indicator_list is not None and indicator_list != "": - if indicator_list == "all": - print(bigdf) - else: - available_inds = [] - for ind in indicator_list.split(","): - if ind in bigdf: - available_inds.append(ind) - ilist = ["pair", "enter_reason", "exit_reason"] + available_inds - _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) + if "all" in indicator_list: + print(bigdf) + elif indicator_list is not None: + available_inds = [] + for ind in indicator_list: + if ind in bigdf: + available_inds.append(ind) + ilist = ["pair", "enter_reason", "exit_reason"] + available_inds + _print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False) else: print("\\_ No trades to show") @@ -205,10 +201,10 @@ def _print_table(df, sortcols=None, show_index=False): def process_entry_exit_reasons(backtest_dir: Path, pairlist: List[str], - analysis_groups: Optional[str] = "0,1,2", - enter_reason_list: Optional[str] = "all", - exit_reason_list: Optional[str] = "all", - indicator_list: Optional[str] = None): + analysis_groups: Optional[List[str]] = ["0", "1", "2"], + enter_reason_list: Optional[List[str]] = ["all"], + exit_reason_list: Optional[List[str]] = ["all"], + indicator_list: Optional[List[str]] = []): try: backtest_stats = load_backtest_stats(backtest_dir) for strategy_name, results in backtest_stats['strategy'].items(): From 4a5ed5a2731bc78522c2ed3118398f64f538975e Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 15 Jun 2022 11:48:57 +0100 Subject: [PATCH 24/25] Fix tests --- tests/data/test_entryexitanalysis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py index 6209110fe..09fbe9957 100755 --- a/tests/data/test_entryexitanalysis.py +++ b/tests/data/test_entryexitanalysis.py @@ -103,8 +103,8 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp # test group 0 and indicator list args = get_args(base_args + - ['--analysis-groups', '0', - '--indicator-list', 'close,rsi,profit_abs'] + ['--analysis-groups', "0", + '--indicator-list', "close", "rsi", "profit_abs"] ) start_analysis_entries_exits(args) captured = capsys.readouterr() @@ -128,7 +128,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '47.0996' in captured.out # test group 1 - args = get_args(base_args + ['--analysis-groups', '1']) + args = get_args(base_args + ['--analysis-groups', "1"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -141,7 +141,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '0' in captured.out # test group 2 - args = get_args(base_args + ['--analysis-groups', '2']) + args = get_args(base_args + ['--analysis-groups', "2"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'enter_tag_long_a' in captured.out @@ -156,7 +156,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2.5' in captured.out # test group 3 - args = get_args(base_args + ['--analysis-groups', '3']) + args = get_args(base_args + ['--analysis-groups', "3"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out @@ -171,7 +171,7 @@ def test_backtest_analysis_nomock(default_conf, mocker, caplog, testdatadir, tmp assert '2' in captured.out # test group 4 - args = get_args(base_args + ['--analysis-groups', '4']) + args = get_args(base_args + ['--analysis-groups', "4"]) start_analysis_entries_exits(args) captured = capsys.readouterr() assert 'LTC/BTC' in captured.out From e2e6c790be9869e8bb82d9bf58f8a2d8f98c0cec Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Jun 2022 16:50:25 +0200 Subject: [PATCH 25/25] Minor doc update --- docs/advanced-backtesting.md | 2 +- docs/utils.md | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md index b6b75c47d..5c2500f18 100644 --- a/docs/advanced-backtesting.md +++ b/docs/advanced-backtesting.md @@ -51,7 +51,7 @@ More options are available by running with the `-h` option. Normally, `backtesting-analysis` uses the latest backtest results, but if you wanted to go back to a previous backtest output, you need to supply the `--export-filename` option. You can supply the same parameter to `backtest-analysis` with the name of the final backtest -output file. This allows you to keep historical versions of backtest results and reanalyse +output file. This allows you to keep historical versions of backtest results and re-analyse them at a later date: ``` bash diff --git a/docs/utils.md b/docs/utils.md index f87aa2ffc..0dd88b242 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -660,26 +660,31 @@ More details in the [Backtesting analysis](advanced-backtesting.md#analyze-the-b ``` usage: freqtrade backtesting-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] - [--analysis-groups [ANALYSIS_GROUPS]] - [--enter-reason-list [ENTER_REASON_LIST]] - [--exit-reason-list [EXIT_REASON_LIST]] - [--indicator-list [INDICATOR_LIST]] + [--export-filename PATH] + [--analysis-groups {0,1,2,3,4} [{0,1,2,3,4} ...]] + [--enter-reason-list ENTER_REASON_LIST [ENTER_REASON_LIST ...]] + [--exit-reason-list EXIT_REASON_LIST [EXIT_REASON_LIST ...]] + [--indicator-list INDICATOR_LIST [INDICATOR_LIST ...]] optional arguments: -h, --help show this help message and exit - --analysis-groups [ANALYSIS_GROUPS] + --export-filename PATH, --backtest-filename PATH + Use this filename for backtest results.Requires + `--export` to be set as well. Example: `--export-filen + ame=user_data/backtest_results/backtest_today.json` + --analysis-groups {0,1,2,3,4} [{0,1,2,3,4} ...] grouping output - 0: simple wins/losses by enter tag, 1: by enter_tag, 2: by enter_tag and exit_tag, 3: by pair and enter_tag, 4: by pair, enter_ and exit_tag (this can get quite large) - --enter-reason-list [ENTER_REASON_LIST] + --enter-reason-list ENTER_REASON_LIST [ENTER_REASON_LIST ...] Comma separated list of entry signals to analyse. Default: all. e.g. 'entry_tag_a,entry_tag_b' - --exit-reason-list [EXIT_REASON_LIST] + --exit-reason-list EXIT_REASON_LIST [EXIT_REASON_LIST ...] Comma separated list of exit signals to analyse. Default: all. e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss' - --indicator-list [INDICATOR_LIST] + --indicator-list INDICATOR_LIST [INDICATOR_LIST ...] Comma separated list of indicators to analyse. e.g. 'close,rsi,bb_lowerband,profit_abs'