From cd2520603ad074256ccae76c380e499000e624fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:07:01 +0100 Subject: [PATCH 01/21] refactor: move json file dumping to separate method --- freqtrade/misc.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 5ea227984..f6287b756 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -19,6 +19,15 @@ from freqtrade.enums import SignalTagType, SignalType logger = logging.getLogger(__name__) +def dump_json_to_file(file_obj: TextIO, data: Any) -> None: + """ + Dump JSON data into a file object + :param file_obj: File object to write to + :param data: JSON Data to save + """ + rapidjson.dump(data, file_obj, default=str, number_mode=rapidjson.NM_NATIVE) + + def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None: """ Dump JSON data into a file @@ -35,12 +44,12 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.info(f'dumping json to "{filename}"') with gzip.open(filename, "wt", encoding="utf-8") as fpz: - rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE) + dump_json_to_file(fpz, data) else: if log: logger.info(f'dumping json to "{filename}"') with filename.open("w") as fp: - rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) + dump_json_to_file(fp, data) logger.debug(f'done json to "{filename}"') From 2a9a3e020c6999d3ee24d13e065c8022f4156394 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:09:18 +0100 Subject: [PATCH 02/21] chore: move joblib export to bt_storage function --- freqtrade/misc.py | 16 ---------------- .../optimize/optimize_reports/bt_storage.py | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index f6287b756..a6dbed85e 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -54,22 +54,6 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = logger.debug(f'done json to "{filename}"') -def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: - """ - Dump object data into a file - :param filename: file to create - :param data: Object data to save - :return: - """ - import joblib - - if log: - logger.info(f'dumping joblib to "{filename}"') - with filename.open("wb") as fp: - joblib.dump(data, fp) - logger.debug(f'done joblib dump to "{filename}"') - - def json_load(datafile: TextIO) -> Any: """ load data with rapidjson diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index cd47aab18..79eca930e 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -1,18 +1,35 @@ import logging from pathlib import Path +from typing import Any from pandas import DataFrame from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.enums.runmode import RunMode from freqtrade.ft_types import BacktestResultType -from freqtrade.misc import file_dump_joblib, file_dump_json +from freqtrade.misc import file_dump_json from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) +def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: + """ + Dump object data into a file + :param filename: file to create + :param data: Object data to save + :return: + """ + import joblib + + if log: + logger.info(f'dumping joblib to "{filename}"') + with filename.open("wb") as fp: + joblib.dump(data, fp) + logger.debug(f'done joblib dump to "{filename}"') + + def _generate_filename(recordfilename: Path, appendix: str, suffix: str) -> Path: """ Generates a filename based on the provided parameters. From 37bbb0e0e09404871ec84a81747cf944c8bd185e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:14:46 +0100 Subject: [PATCH 03/21] refactor: move joblib_writer to use file interface --- freqtrade/optimize/optimize_reports/bt_storage.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index 79eca930e..479588a8d 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any +from typing import Any, TextIO from pandas import DataFrame @@ -14,7 +14,7 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) -def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: +def file_dump_joblib(file_obj: TextIO, data: Any, log: bool = True) -> None: """ Dump object data into a file :param filename: file to create @@ -23,11 +23,7 @@ def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None: """ import joblib - if log: - logger.info(f'dumping joblib to "{filename}"') - with filename.open("wb") as fp: - joblib.dump(data, fp) - logger.debug(f'done joblib dump to "{filename}"') + joblib.dump(data, file_obj) def _generate_filename(recordfilename: Path, appendix: str, suffix: str) -> Path: @@ -122,6 +118,9 @@ def _store_backtest_analysis_data( """ filename = _generate_filename(recordfilename, f"{dtappendix}_{name}", ".pkl") - file_dump_joblib(filename, data) + logger.info(f'dumping joblib to "{filename}"') + with filename.open("wb") as fp: + file_dump_joblib(fp, data) + logger.debug(f'done joblib dump to "{filename}"') return filename From b1fc271a7e446eb1816e6258f3f7f2627f3d5019 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:28:18 +0100 Subject: [PATCH 04/21] chore: Update storage to store backtest results as zip file this will enable further additions to files without polluting the filesystem further. --- .../optimize/optimize_reports/bt_storage.py | 106 ++++++++---------- 1 file changed, 46 insertions(+), 60 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index 479588a8d..69bea4883 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -1,4 +1,6 @@ import logging +import zipfile +from io import BytesIO, StringIO from pathlib import Path from typing import Any, TextIO @@ -7,7 +9,7 @@ from pandas import DataFrame from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.enums.runmode import RunMode from freqtrade.ft_types import BacktestResultType -from freqtrade.misc import file_dump_json +from freqtrade.misc import dump_json_to_file, file_dump_json from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename @@ -52,75 +54,59 @@ def store_backtest_results( analysis_results: dict[str, dict[str, DataFrame]] | None = None, ) -> Path: """ - Stores backtest results and analysis data + Stores backtest results and analysis data in a zip file, with metadata stored separately + for convenience. :param config: Configuration dictionary :param stats: Dataframe containing the backtesting statistics :param dtappendix: Datetime to use for the filename :param market_change_data: Dataframe containing market change data :param analysis_results: Dictionary containing analysis results """ - - # Path object, which can either be a filename or a directory. - # Filenames will be appended with a timestamp right before the suffix - # while for directories, /backtest-result-.json will be used as filename recordfilename: Path = config["exportfilename"] - filename = _generate_filename(recordfilename, dtappendix, ".json") + zip_filename = _generate_filename(recordfilename, dtappendix, ".zip") + base_filename = _generate_filename(recordfilename, dtappendix, "") + json_filename = _generate_filename(recordfilename, dtappendix, ".json") - # Store metadata separately. - file_dump_json(get_backtest_metadata_filename(filename), stats["metadata"]) - # Don't mutate the original stats dict. - stats_copy = { - "strategy": stats["strategy"], - "strategy_comparison": stats["strategy_comparison"], - } + # Store metadata separately with .json extension + file_dump_json(get_backtest_metadata_filename(json_filename), stats["metadata"]) - file_dump_json(filename, stats_copy) + # Store latest backtest info separately + latest_filename = Path.joinpath(zip_filename.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {"latest_backtest": str(zip_filename.name)}) - latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) - file_dump_json(latest_filename, {"latest_backtest": str(filename.name)}) + # Create zip file and add the files + with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: + # Store stats + stats_copy = { + "strategy": stats["strategy"], + "strategy_comparison": stats["strategy_comparison"], + } + stats_buf = StringIO() + dump_json_to_file(stats_buf, stats_copy) + zipf.writestr(json_filename, stats_buf.getvalue()) - if market_change_data is not None: - filename_mc = _generate_filename(recordfilename, f"{dtappendix}_market_change", ".feather") - market_change_data.reset_index().to_feather( - filename_mc, compression_level=9, compression="lz4" - ) + # Add market change data if present + if market_change_data is not None: + market_change_name = f"{base_filename.stem}_market_change.feather" + market_change_buf = BytesIO() + market_change_data.reset_index().to_feather( + market_change_buf, compression_level=9, compression="lz4" + ) + market_change_buf.seek(0) + zipf.writestr(market_change_name, market_change_buf.getvalue()) - if ( - config.get("export", "none") == "signals" - and analysis_results is not None - and config.get("runmode", RunMode.OTHER) == RunMode.BACKTEST - ): - _store_backtest_analysis_data( - recordfilename, analysis_results["signals"], dtappendix, "signals" - ) - _store_backtest_analysis_data( - recordfilename, analysis_results["rejected"], dtappendix, "rejected" - ) - _store_backtest_analysis_data( - recordfilename, analysis_results["exited"], dtappendix, "exited" - ) + # Add analysis results if present and running in backtest mode + if ( + config.get("export", "none") == "signals" + and analysis_results is not None + and config.get("runmode", RunMode.OTHER) == RunMode.BACKTEST + ): + for name in ["signals", "rejected", "exited"]: + if name in analysis_results: + analysis_name = f"{base_filename.stem}_{name}.pkl" + analysis_buf = BytesIO() + file_dump_joblib(analysis_buf, analysis_results[name]) + analysis_buf.seek(0) + zipf.writestr(analysis_name, analysis_buf.getvalue()) - return filename - - -def _store_backtest_analysis_data( - recordfilename: Path, data: dict[str, dict], dtappendix: str, name: str -) -> Path: - """ - Stores backtest trade candles for analysis - :param recordfilename: Path object, which can either be a filename or a directory. - Filenames will be appended with a timestamp right before the suffix - while for directories, /backtest-result-_.pkl will be used - as filename - :param candles: Dict containing the backtesting data for analysis - :param dtappendix: Datetime to use for the filename - :param name: Name to use for the file, e.g. signals, rejected - """ - filename = _generate_filename(recordfilename, f"{dtappendix}_{name}", ".pkl") - - logger.info(f'dumping joblib to "{filename}"') - with filename.open("wb") as fp: - file_dump_joblib(fp, data) - logger.debug(f'done joblib dump to "{filename}"') - - return filename + return zip_filename From d2d6c6fa8111266e2eced5b79ec4c80fec1ae332 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:42:32 +0100 Subject: [PATCH 05/21] feat: load backtest-results from either file or zip file --- freqtrade/data/btanalysis.py | 21 +++++++++++++++++-- .../optimize/optimize_reports/bt_storage.py | 4 ++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index a9d8648d2..c3d3255fa 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,8 +3,10 @@ Helpers when analyzing backtest data """ import logging +import zipfile from copy import copy from datetime import datetime, timezone +from io import StringIO from pathlib import Path from typing import Any, Literal @@ -165,8 +167,23 @@ def load_backtest_stats(filename: Path | str) -> BacktestResultType: if not filename.is_file(): raise ValueError(f"File {filename} does not exist.") logger.info(f"Loading backtest result from {filename}") - with filename.open() as file: - data = json_load(file) + + if filename.suffix == ".zip": + try: + with zipfile.ZipFile(filename) as zipf: + json_filename = filename.with_suffix(".json") + try: + with zipf.open(json_filename.name) as json_file: + # Need to convert to StringIO since json_load expects a text stream + data = json_load(StringIO(json_file.read().decode("utf-8"))) + except KeyError: + # File not found in zip + raise ValueError(f"Could not find {json_filename.name} in {filename}") + except zipfile.BadZipFile: + raise ValueError(f"Bad zip file: {filename}") + else: + with filename.open() as file: + data = json_load(file) # Legacy list format does not contain metadata. if isinstance(data, dict): diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index 69bea4883..b2f613779 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -72,7 +72,7 @@ def store_backtest_results( # Store latest backtest info separately latest_filename = Path.joinpath(zip_filename.parent, LAST_BT_RESULT_FN) - file_dump_json(latest_filename, {"latest_backtest": str(zip_filename.name)}) + file_dump_json(latest_filename, {"latest_backtest": str(zip_filename.name)}, log=False) # Create zip file and add the files with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: @@ -83,7 +83,7 @@ def store_backtest_results( } stats_buf = StringIO() dump_json_to_file(stats_buf, stats_copy) - zipf.writestr(json_filename, stats_buf.getvalue()) + zipf.writestr(json_filename.name, stats_buf.getvalue()) # Add market change data if present if market_change_data is not None: From b1bf56cfab50512d77c60a9673811cdb9ef6711d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Dec 2024 19:47:56 +0100 Subject: [PATCH 06/21] chore: use exception logging for exceptions --- freqtrade/data/entryexitanalysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index b4b5b966a..47a845e79 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -32,8 +32,8 @@ def _load_backtest_analysis_data(backtest_dir: Path, name: str): with scpf.open("rb") as scp: loaded_data = joblib.load(scp) logger.info(f"Loaded {name} candles: {str(scpf)}") - except Exception as e: - logger.error(f"Cannot load {name} data from pickled results: ", e) + except Exception: + logger.exception(f"Cannot load {name} data from pickled results.") return None return loaded_data From 258b85a860d4a4293e6a23cb36b91da3819d5ac2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 15:18:13 +0100 Subject: [PATCH 07/21] feat: load backtest-analysis-data files from zipfile --- freqtrade/data/entryexitanalysis.py | 56 ++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 47a845e79..7ae8650e9 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,4 +1,5 @@ import logging +import zipfile from pathlib import Path import joblib @@ -20,23 +21,52 @@ logger = logging.getLogger(__name__) def _load_backtest_analysis_data(backtest_dir: Path, name: str): + """ + Load backtest analysis data either from a pickle file or from within a zip file + :param backtest_dir: Directory containing backtest results + :param name: Name of the analysis data to load (signals, rejected, exited) + :return: Analysis data + """ if backtest_dir.is_dir(): - scpf = Path( - backtest_dir, - Path(get_latest_backtest_filename(backtest_dir)).stem + "_" + name + ".pkl", - ) + lbf = Path(get_latest_backtest_filename(backtest_dir)) + zip_path = backtest_dir / lbf else: - scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl") + zip_path = backtest_dir - try: - with scpf.open("rb") as scp: - loaded_data = joblib.load(scp) - logger.info(f"Loaded {name} candles: {str(scpf)}") - except Exception: - logger.exception(f"Cannot load {name} data from pickled results.") - return None + if zip_path.suffix == ".zip": + # Load from zip file + try: + with zipfile.ZipFile(zip_path) as zipf: + # Files in zip are stored with just their base names + analysis_name = f"{zip_path.stem}_{name}.pkl" + try: + with zipf.open(analysis_name) as analysis_file: + loaded_data = joblib.load(analysis_file) + logger.info( + f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}" + ) + return loaded_data + except KeyError: + logger.exception(f"Cannot find {analysis_name} in {zip_path}") + return None + except zipfile.BadZipFile: + logger.exception(f"Bad zip file: {zip_path}") + return None + else: + # Load from separate pickle file + if backtest_dir.is_dir(): + scpf = Path(backtest_dir, f"{zip_path.stem}_{name}.pkl") + else: + scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl") - return loaded_data + try: + with scpf.open("rb") as scp: + loaded_data = joblib.load(scp) + logger.info(f"Loaded {name} candles: {str(scpf)}") + return loaded_data + except Exception: + logger.exception(f"Cannot load {name} data from pickled results.") + return None def _load_rejected_signals(backtest_dir: Path): From 9f2142135ee9ce351865c7de5a98141414184193 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 15:22:57 +0100 Subject: [PATCH 08/21] refactor: move load-analysis-data logic to btanalysis file --- freqtrade/data/btanalysis.py | 72 ++++++++++++++++++++++++++++ freqtrade/data/entryexitanalysis.py | 73 +++-------------------------- 2 files changed, 78 insertions(+), 67 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index c3d3255fa..09045528f 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -405,6 +405,78 @@ def load_backtest_data(filename: Path | str, strategy: str | None = None) -> pd. return df +def load_backtest_analysis_data(backtest_dir: Path, name: str): + """ + Load backtest analysis data either from a pickle file or from within a zip file + :param backtest_dir: Directory containing backtest results + :param name: Name of the analysis data to load (signals, rejected, exited) + :return: Analysis data + """ + import joblib + + if backtest_dir.is_dir(): + lbf = Path(get_latest_backtest_filename(backtest_dir)) + zip_path = backtest_dir / lbf + else: + zip_path = backtest_dir + + if zip_path.suffix == ".zip": + # Load from zip file + try: + with zipfile.ZipFile(zip_path) as zipf: + # Files in zip are stored with just their base names + analysis_name = f"{zip_path.stem}_{name}.pkl" + try: + with zipf.open(analysis_name) as analysis_file: + loaded_data = joblib.load(analysis_file) + logger.info( + f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}" + ) + return loaded_data + except KeyError: + logger.exception(f"Cannot find {analysis_name} in {zip_path}") + return None + except zipfile.BadZipFile: + logger.exception(f"Bad zip file: {zip_path}") + return None + else: + # Load from separate pickle file + if backtest_dir.is_dir(): + scpf = Path(backtest_dir, f"{zip_path.stem}_{name}.pkl") + else: + scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl") + + try: + with scpf.open("rb") as scp: + loaded_data = joblib.load(scp) + logger.info(f"Loaded {name} candles: {str(scpf)}") + return loaded_data + except Exception: + logger.exception(f"Cannot load {name} data from pickled results.") + return None + + +def load_rejected_signals(backtest_dir: Path): + """ + Load rejected signals from backtest directory + """ + return load_backtest_analysis_data(backtest_dir, "rejected") + + +def load_signal_candles(backtest_dir: Path): + """ + Load signal candles from backtest directory + """ + return load_backtest_analysis_data(backtest_dir, "signals") + + +def load_exit_signal_candles(backtest_dir: Path) -> dict[str, dict[str, pd.DataFrame]]: + """ + Load exit signal candles from backtest directory + """ + return load_backtest_analysis_data(backtest_dir, "exited") + + def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame: """ Find overlapping trades by expanding each trade once per period it was open diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 7ae8650e9..73eda2f7d 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -1,17 +1,17 @@ import logging -import zipfile from pathlib import Path -import joblib import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import Config from freqtrade.data.btanalysis import ( BT_DATA_COLUMNS, - get_latest_backtest_filename, load_backtest_data, load_backtest_stats, + load_exit_signal_candles, + load_rejected_signals, + load_signal_candles, ) from freqtrade.exceptions import OperationalException from freqtrade.util import print_df_rich_table @@ -20,67 +20,6 @@ from freqtrade.util import print_df_rich_table logger = logging.getLogger(__name__) -def _load_backtest_analysis_data(backtest_dir: Path, name: str): - """ - Load backtest analysis data either from a pickle file or from within a zip file - :param backtest_dir: Directory containing backtest results - :param name: Name of the analysis data to load (signals, rejected, exited) - :return: Analysis data - """ - if backtest_dir.is_dir(): - lbf = Path(get_latest_backtest_filename(backtest_dir)) - zip_path = backtest_dir / lbf - else: - zip_path = backtest_dir - - if zip_path.suffix == ".zip": - # Load from zip file - try: - with zipfile.ZipFile(zip_path) as zipf: - # Files in zip are stored with just their base names - analysis_name = f"{zip_path.stem}_{name}.pkl" - try: - with zipf.open(analysis_name) as analysis_file: - loaded_data = joblib.load(analysis_file) - logger.info( - f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}" - ) - return loaded_data - except KeyError: - logger.exception(f"Cannot find {analysis_name} in {zip_path}") - return None - except zipfile.BadZipFile: - logger.exception(f"Bad zip file: {zip_path}") - return None - else: - # Load from separate pickle file - if backtest_dir.is_dir(): - scpf = Path(backtest_dir, f"{zip_path.stem}_{name}.pkl") - else: - scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl") - - try: - with scpf.open("rb") as scp: - loaded_data = joblib.load(scp) - logger.info(f"Loaded {name} candles: {str(scpf)}") - return loaded_data - except Exception: - logger.exception(f"Cannot load {name} data from pickled results.") - return None - - -def _load_rejected_signals(backtest_dir: Path): - return _load_backtest_analysis_data(backtest_dir, "rejected") - - -def _load_signal_candles(backtest_dir: Path): - return _load_backtest_analysis_data(backtest_dir, "signals") - - -def _load_exit_signal_candles(backtest_dir: Path) -> dict[str, dict[str, pd.DataFrame]]: - return _load_backtest_analysis_data(backtest_dir, "exited") - - def _process_candles_and_indicators( pairlist, strategy_name, trades, signal_candles, date_col: str = "open_date" ): @@ -411,12 +350,12 @@ def process_entry_exit_reasons(config: Config): trades = load_backtest_data(config["exportfilename"], strategy_name) if trades is not None and not trades.empty: - signal_candles = _load_signal_candles(config["exportfilename"]) - exit_signals = _load_exit_signal_candles(config["exportfilename"]) + signal_candles = load_signal_candles(config["exportfilename"]) + exit_signals = load_exit_signal_candles(config["exportfilename"]) rej_df = None if do_rejected: - rejected_signals_dict = _load_rejected_signals(config["exportfilename"]) + rejected_signals_dict = load_rejected_signals(config["exportfilename"]) rej_df = prepare_results( rejected_signals_dict, strategy_name, From 2768155e7d2da8127331adc57680d5c788fd5d33 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 15:49:33 +0100 Subject: [PATCH 09/21] fix: use .json metadata file --- freqtrade/optimize/backtest_caching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtest_caching.py b/freqtrade/optimize/backtest_caching.py index 7143da006..9c3e2f343 100644 --- a/freqtrade/optimize/backtest_caching.py +++ b/freqtrade/optimize/backtest_caching.py @@ -40,4 +40,4 @@ def get_strategy_run_id(strategy) -> str: def get_backtest_metadata_filename(filename: Path | str) -> Path: """Return metadata filename for specified backtest results file.""" filename = Path(filename) - return filename.parent / Path(f"{filename.stem}.meta{filename.suffix}") + return filename.parent / Path(f"{filename.stem}.meta.json") From b08a93a287492550274722cdac1aae3eeeb74693 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 15:49:47 +0100 Subject: [PATCH 10/21] feat: Detect both .json and .zip backtest results --- freqtrade/data/btanalysis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 09045528f..0595fa8e2 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -211,8 +211,10 @@ def load_and_merge_backtest_result(strategy_name: str, filename: Path, results: def _get_backtest_files(dirname: Path) -> list[Path]: - # Weird glob expression here avoids including .meta.json files. - return list(reversed(sorted(dirname.glob("backtest-result-*-[0-9][0-9].json")))) + # Get both json and zip files separately and combine the results + json_files = dirname.glob("backtest-result-*-[0-9][0-9]*.json") + zip_files = dirname.glob("backtest-result-*-[0-9][0-9]*.zip") + return list(reversed(sorted(list(json_files) + list(zip_files)))) def _extract_backtest_result(filename: Path) -> list[BacktestHistoryEntryType]: From a88675b6930cd4de81cf3704eb8fbeb475b852b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 15:53:44 +0100 Subject: [PATCH 11/21] test: improve test_metadata_filename test --- tests/optimize/test_backtesting.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 23922ee42..556ec7f68 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -2660,7 +2660,7 @@ def test_get_backtest_metadata_filename(): # Test with a string file path with no extension filename = "/path/to/backtest_results" - expected = Path("/path/to/backtest_results.meta") + expected = Path("/path/to/backtest_results.meta.json") assert get_backtest_metadata_filename(filename) == expected # Test with a string file path with multiple dots in the name @@ -2672,3 +2672,8 @@ def test_get_backtest_metadata_filename(): filename = "backtest_results.json" expected = Path("backtest_results.meta.json") assert get_backtest_metadata_filename(filename) == expected + # Test with a string file path with no parent directory + + filename = "backtest_results_zip.zip" + expected = Path("backtest_results_zip.meta.json") + assert get_backtest_metadata_filename(filename) == expected From 606013a697331f19e58da569b5ad0da2372d54a7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:06:55 +0100 Subject: [PATCH 12/21] chore: import ZipFile directly --- freqtrade/optimize/optimize_reports/bt_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index b2f613779..b6b110638 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -1,8 +1,8 @@ import logging -import zipfile from io import BytesIO, StringIO from pathlib import Path from typing import Any, TextIO +from zipfile import ZIP_DEFLATED, ZipFile from pandas import DataFrame @@ -75,7 +75,7 @@ def store_backtest_results( file_dump_json(latest_filename, {"latest_backtest": str(zip_filename.name)}, log=False) # Create zip file and add the files - with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: + with ZipFile(zip_filename, "w", ZIP_DEFLATED) as zipf: # Store stats stats_copy = { "strategy": stats["strategy"], From aa99baf268c59d89dbfbab1c41c706cfc99f43b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:07:01 +0100 Subject: [PATCH 13/21] test: update a few tests --- tests/optimize/test_optimize_reports.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 77fb10655..56c72531a 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,3 +1,4 @@ +import json import re from datetime import timedelta from pathlib import Path @@ -229,13 +230,14 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmp_path): store_backtest_results(default_conf, stats, "2022_01_01_15_05_13") - # get real Filename (it's btresult-.json) + # get real Filename (it's btresult-.zip) last_fn = get_latest_backtest_filename(filename_last.parent) - assert re.match(r"btresult-.*\.json", last_fn) + assert re.match(r"btresult-.*\.zip", last_fn) filename1 = tmp_path / last_fn assert filename1.is_file() - content = filename1.read_text() + + content = json.dumps(load_backtest_stats(filename1)) assert "max_drawdown_account" in content assert "strategy" in content assert "pairlist" in content @@ -248,19 +250,22 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmp_path): def test_store_backtest_results(testdatadir, mocker): dump_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_json") - + zip_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.ZipFile") data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} store_backtest_results({"exportfilename": testdatadir}, data, "2022_01_01_15_05_13") - assert dump_mock.call_count == 3 + assert dump_mock.call_count == 2 + assert zip_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / "backtest-result")) dump_mock.reset_mock() + zip_mock.reset_mock() filename = testdatadir / "testresult.json" store_backtest_results({"exportfilename": filename}, data, "2022_01_01_15_05_13") - assert dump_mock.call_count == 3 + assert dump_mock.call_count == 2 + assert zip_mock.call_count == 1 assert isinstance(dump_mock.call_args_list[0][0][0], Path) # result will be testdatadir / testresult-.json assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / "testresult")) From bb372e380ddbacc76609801504ad836762dd4ef8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:19:46 +0100 Subject: [PATCH 14/21] test: clean up optimize_report tests --- tests/optimize/test_optimize_reports.py | 106 +++++++++++------------- 1 file changed, 48 insertions(+), 58 deletions(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 56c72531a..5459f76f5 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -3,6 +3,7 @@ import re from datetime import timedelta from pathlib import Path from shutil import copyfile +from zipfile import ZipFile import joblib import pandas as pd @@ -275,67 +276,33 @@ def test_store_backtest_results_real(tmp_path): data = {"metadata": {}, "strategy": {}, "strategy_comparison": []} store_backtest_results({"exportfilename": tmp_path}, data, "2022_01_01_15_05_13") - assert (tmp_path / "backtest-result-2022_01_01_15_05_13.json").is_file() + zip_file = tmp_path / "backtest-result-2022_01_01_15_05_13.zip" + assert zip_file.is_file() assert (tmp_path / "backtest-result-2022_01_01_15_05_13.meta.json").is_file() assert not (tmp_path / "backtest-result-2022_01_01_15_05_13_market_change.feather").is_file() + with ZipFile(zip_file, "r") as zipf: + assert "backtest-result-2022_01_01_15_05_13.json" in zipf.namelist() + assert "backtest-result-2022_01_01_15_05_13_market_change.feather" not in zipf.namelist() assert (tmp_path / LAST_BT_RESULT_FN).is_file() fn = get_latest_backtest_filename(tmp_path) - assert fn == "backtest-result-2022_01_01_15_05_13.json" + assert fn == "backtest-result-2022_01_01_15_05_13.zip" store_backtest_results( {"exportfilename": tmp_path}, data, "2024_01_01_15_05_25", market_change_data=pd.DataFrame() ) - assert (tmp_path / "backtest-result-2024_01_01_15_05_25.json").is_file() + zip_file = tmp_path / "backtest-result-2024_01_01_15_05_25.zip" + assert zip_file.is_file() assert (tmp_path / "backtest-result-2024_01_01_15_05_25.meta.json").is_file() - assert (tmp_path / "backtest-result-2024_01_01_15_05_25_market_change.feather").is_file() + assert not (tmp_path / "backtest-result-2024_01_01_15_05_25_market_change.feather").is_file() + + with ZipFile(zip_file, "r") as zipf: + assert "backtest-result-2024_01_01_15_05_25.json" in zipf.namelist() + assert "backtest-result-2024_01_01_15_05_25_market_change.feather" in zipf.namelist() assert (tmp_path / LAST_BT_RESULT_FN).is_file() # Last file reference should be updated fn = get_latest_backtest_filename(tmp_path) - assert fn == "backtest-result-2024_01_01_15_05_25.json" - - -def test_store_backtest_candles(tmp_path, mocker): - mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_json") - dump_mock = mocker.patch("freqtrade.optimize.optimize_reports.bt_storage.file_dump_joblib") - - candle_dict = {"DefStrat": {"UNITTEST/BTC": pd.DataFrame()}} - bt_results = {"metadata": {}, "strategy": {}, "strategy_comparison": []} - - mock_conf = { - "exportfilename": tmp_path, - "export": "signals", - "runmode": "backtest", - } - - # mock directory exporting - data = { - "signals": candle_dict, - "rejected": {}, - "exited": {}, - } - - store_backtest_results(mock_conf, bt_results, "2022_01_01_15_05_13", analysis_results=data) - - assert dump_mock.call_count == 3 - assert isinstance(dump_mock.call_args_list[0][0][0], Path) - assert str(dump_mock.call_args_list[0][0][0]).endswith("_signals.pkl") - assert str(dump_mock.call_args_list[1][0][0]).endswith("_rejected.pkl") - assert str(dump_mock.call_args_list[2][0][0]).endswith("_exited.pkl") - - dump_mock.reset_mock() - # mock file exporting - filename = Path(tmp_path / "testresult") - mock_conf["exportfilename"] = filename - store_backtest_results(mock_conf, bt_results, "2022_01_01_15_05_13", analysis_results=data) - assert dump_mock.call_count == 3 - assert isinstance(dump_mock.call_args_list[0][0][0], Path) - # result will be tmp_path / testresult-_signals.pkl - assert str(dump_mock.call_args_list[0][0][0]).endswith("_signals.pkl") - assert str(dump_mock.call_args_list[1][0][0]).endswith("_rejected.pkl") - assert str(dump_mock.call_args_list[2][0][0]).endswith("_exited.pkl") - - dump_mock.reset_mock() + assert fn == "backtest-result-2024_01_01_15_05_25.zip" def test_write_read_backtest_candles(tmp_path): @@ -355,9 +322,21 @@ def test_write_read_backtest_candles(tmp_path): "exited": {}, } store_backtest_results(mock_conf, bt_results, sample_date, analysis_results=data) - stored_file = tmp_path / f"backtest-result-{sample_date}_signals.pkl" - with stored_file.open("rb") as scp: - pickled_signal_candles = joblib.load(scp) + stored_file = tmp_path / f"backtest-result-{sample_date}.zip" + signals_pkl = f"backtest-result-{sample_date}_signals.pkl" + rejected_pkl = f"backtest-result-{sample_date}_rejected.pkl" + exited_pkl = f"backtest-result-{sample_date}_exited.pkl" + assert not (tmp_path / signals_pkl).is_file() + assert stored_file.is_file() + + with ZipFile(stored_file, "r") as zipf: + assert signals_pkl in zipf.namelist() + assert rejected_pkl in zipf.namelist() + assert exited_pkl in zipf.namelist() + + # open and read the file + with zipf.open(signals_pkl) as scp: + pickled_signal_candles = joblib.load(scp) assert pickled_signal_candles.keys() == candle_dict.keys() assert pickled_signal_candles["DefStrat"].keys() == pickled_signal_candles["DefStrat"].keys() @@ -371,14 +350,25 @@ def test_write_read_backtest_candles(tmp_path): filename = tmp_path / "testresult" mock_conf["exportfilename"] = filename store_backtest_results(mock_conf, bt_results, sample_date, analysis_results=data) - stored_file = tmp_path / f"testresult-{sample_date}_signals.pkl" - with stored_file.open("rb") as scp: - pickled_signal_candles = joblib.load(scp) + stored_file = tmp_path / f"testresult-{sample_date}.zip" + signals_pkl = f"testresult-{sample_date}_signals.pkl" + rejected_pkl = f"testresult-{sample_date}_rejected.pkl" + exited_pkl = f"testresult-{sample_date}_exited.pkl" + assert not (tmp_path / signals_pkl).is_file() + assert stored_file.is_file() - assert pickled_signal_candles.keys() == candle_dict.keys() - assert pickled_signal_candles["DefStrat"].keys() == pickled_signal_candles["DefStrat"].keys() - assert pickled_signal_candles["DefStrat"]["UNITTEST/BTC"].equals( - pickled_signal_candles["DefStrat"]["UNITTEST/BTC"] + with ZipFile(stored_file, "r") as zipf: + assert signals_pkl in zipf.namelist() + assert rejected_pkl in zipf.namelist() + assert exited_pkl in zipf.namelist() + + with zipf.open(signals_pkl) as scp: + pickled_signal_candles2 = joblib.load(scp) + + assert pickled_signal_candles2.keys() == candle_dict.keys() + assert pickled_signal_candles2["DefStrat"].keys() == pickled_signal_candles2["DefStrat"].keys() + assert pickled_signal_candles2["DefStrat"]["UNITTEST/BTC"].equals( + pickled_signal_candles2["DefStrat"]["UNITTEST/BTC"] ) _clean_test_file(stored_file) From a40b2816a19606e22699ea38b7b1be23d75346d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:26:02 +0100 Subject: [PATCH 15/21] chore: greatly simplify analyze_commands --- freqtrade/commands/analyze_commands.py | 44 ++------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py index 7966ae6e6..d76abd1ef 100644 --- a/freqtrade/commands/analyze_commands.py +++ b/freqtrade/commands/analyze_commands.py @@ -1,63 +1,23 @@ import logging -from pathlib import Path from typing import Any from freqtrade.enums import RunMode -from freqtrade.exceptions import ConfigurationError, OperationalException logger = logging.getLogger(__name__) -def setup_analyze_configuration(args: dict[str, Any], method: RunMode) -> dict[str, Any]: - """ - Prepare the configuration for the entry/exit reason analysis module - :param args: Cli args from Arguments() - :param method: Bot running mode - :return: Configuration - """ - from freqtrade.configuration import setup_utils_configuration - - config = setup_utils_configuration(args, method) - - no_unlimited_runmodes = { - RunMode.BACKTEST: "backtesting", - } - if method in no_unlimited_runmodes.keys(): - from freqtrade.data.btanalysis import get_latest_backtest_filename - - if "exportfilename" in config: - if config["exportfilename"].is_dir(): - btfile = Path(get_latest_backtest_filename(config["exportfilename"])) - signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl" - else: - if config["exportfilename"].exists(): - btfile = Path(config["exportfilename"]) - signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl" - else: - raise ConfigurationError(f"{config['exportfilename']} does not exist.") - else: - raise ConfigurationError("exportfilename not in config.") - - if not Path(signals_file).exists(): - raise OperationalException( - f"Cannot find latest backtest signals file: {signals_file}." - "Run backtesting with `--export signals`." - ) - - return config - - def start_analysis_entries_exits(args: dict[str, Any]) -> None: """ Start analysis script :param args: Cli args from Arguments() :return: None """ + from freqtrade.configuration import setup_utils_configuration from freqtrade.data.entryexitanalysis import process_entry_exit_reasons # Initialize configuration - config = setup_analyze_configuration(args, RunMode.BACKTEST) + config = setup_utils_configuration(args, RunMode.BACKTEST) logger.info("Starting freqtrade in analysis mode") From 6121fb30a926f2d5a614fc82ecb29739a3a05566 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:26:23 +0100 Subject: [PATCH 16/21] chore: improved error-message when exportfilename wasn't found --- freqtrade/data/entryexitanalysis.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index 73eda2f7d..0c49b775c 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -13,7 +13,7 @@ from freqtrade.data.btanalysis import ( load_rejected_signals, load_signal_candles, ) -from freqtrade.exceptions import OperationalException +from freqtrade.exceptions import ConfigurationError, OperationalException from freqtrade.util import print_df_rich_table @@ -343,8 +343,10 @@ def process_entry_exit_reasons(config: Config): timerange = TimeRange.parse_timerange( None if config.get("timerange") is None else str(config.get("timerange")) ) - - backtest_stats = load_backtest_stats(config["exportfilename"]) + try: + backtest_stats = load_backtest_stats(config["exportfilename"]) + except ValueError as e: + raise ConfigurationError(e) from e for strategy_name, results in backtest_stats["strategy"].items(): trades = load_backtest_data(config["exportfilename"], strategy_name) From 8fe1f3cb953f20ccdaa1f40f7402174ca96e6aee Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 16:34:13 +0100 Subject: [PATCH 17/21] feat: load backtest data via API from both zip and json --- freqtrade/rpc/api_server/api_backtest.py | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 65631add6..088a0a700 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -273,15 +273,18 @@ def api_backtest_history(config=Depends(get_config)): def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config)): # Get backtest result history, read from metadata files bt_results_base: Path = config["user_data_dir"] / "backtest_results" - fn = (bt_results_base / filename).with_suffix(".json") + for ext in [".zip", ".json"]: + fn = (bt_results_base / filename).with_suffix(ext) + if is_file_in_dir(fn, bt_results_base): + break + else: + raise HTTPException(status_code=404, detail="File not found.") results: dict[str, Any] = { "metadata": {}, "strategy": {}, "strategy_comparison": [], } - if not is_file_in_dir(fn, bt_results_base): - raise HTTPException(status_code=404, detail="File not found.") load_and_merge_backtest_result(strategy, fn, results) return { "status": "ended", @@ -301,9 +304,12 @@ def api_backtest_history_result(filename: str, strategy: str, config=Depends(get def api_delete_backtest_history_entry(file: str, config=Depends(get_config)): # Get backtest result history, read from metadata files bt_results_base: Path = config["user_data_dir"] / "backtest_results" - file_abs = (bt_results_base / file).with_suffix(".json") - # Ensure file is in backtest_results directory - if not is_file_in_dir(file_abs, bt_results_base): + for ext in [".zip", ".json"]: + file_abs = (bt_results_base / file).with_suffix(ext) + # Ensure file is in backtest_results directory + if is_file_in_dir(file_abs, bt_results_base): + break + else: raise HTTPException(status_code=404, detail="File not found.") delete_backtest_result(file_abs) @@ -320,10 +326,14 @@ def api_update_backtest_history_entry( ): # Get backtest result history, read from metadata files bt_results_base: Path = config["user_data_dir"] / "backtest_results" - file_abs = (bt_results_base / file).with_suffix(".json") - # Ensure file is in backtest_results directory - if not is_file_in_dir(file_abs, bt_results_base): + for ext in [".zip", ".json"]: + file_abs = (bt_results_base / file).with_suffix(ext) + # Ensure file is in backtest_results directory + if is_file_in_dir(file_abs, bt_results_base): + break + else: raise HTTPException(status_code=404, detail="File not found.") + content = {"notes": body.notes} try: update_backtest_metadata(file_abs, body.strategy, content) From 1119ee1cddc4208f5a42d2d7763162941ed1dd53 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 17:01:19 +0100 Subject: [PATCH 18/21] chore: load zip reusability --- freqtrade/data/btanalysis.py | 59 ++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0595fa8e2..49b4b0eb1 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -6,7 +6,7 @@ import logging import zipfile from copy import copy from datetime import datetime, timezone -from io import StringIO +from io import BytesIO, StringIO from pathlib import Path from typing import Any, Literal @@ -169,18 +169,11 @@ def load_backtest_stats(filename: Path | str) -> BacktestResultType: logger.info(f"Loading backtest result from {filename}") if filename.suffix == ".zip": - try: - with zipfile.ZipFile(filename) as zipf: - json_filename = filename.with_suffix(".json") - try: - with zipf.open(json_filename.name) as json_file: - # Need to convert to StringIO since json_load expects a text stream - data = json_load(StringIO(json_file.read().decode("utf-8"))) - except KeyError: - # File not found in zip - raise ValueError(f"Could not find {json_filename.name} in {filename}") - except zipfile.BadZipFile: - raise ValueError(f"Bad zip file: {filename}") + data = json_load( + StringIO( + load_file_from_zip(filename, filename.with_suffix(".json").name).decode("utf-8") + ) + ) else: with filename.open() as file: data = json_load(file) @@ -407,6 +400,22 @@ def load_backtest_data(filename: Path | str, strategy: str | None = None) -> pd. return df +def load_file_from_zip(zip_path: Path, filename: str) -> bytes | None: + """ + Load a file from a zip file + :param zip_path: Path to the zip file + :param filename: Name of the file to load + :return: Bytes of the file + """ + try: + with zipfile.ZipFile(zip_path) as zipf: + with zipf.open(filename) as file: + return file.read() + except zipfile.BadZipFile: + logger.exception(f"Bad zip file: {zip_path}") + return None + + def load_backtest_analysis_data(backtest_dir: Path, name: str): """ Load backtest analysis data either from a pickle file or from within a zip file @@ -424,23 +433,15 @@ def load_backtest_analysis_data(backtest_dir: Path, name: str): if zip_path.suffix == ".zip": # Load from zip file - try: - with zipfile.ZipFile(zip_path) as zipf: - # Files in zip are stored with just their base names - analysis_name = f"{zip_path.stem}_{name}.pkl" - try: - with zipf.open(analysis_name) as analysis_file: - loaded_data = joblib.load(analysis_file) - logger.info( - f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}" - ) - return loaded_data - except KeyError: - logger.exception(f"Cannot find {analysis_name} in {zip_path}") - return None - except zipfile.BadZipFile: - logger.exception(f"Bad zip file: {zip_path}") + analysis_name = f"{zip_path.stem}_{name}.pkl" + data = load_file_from_zip(zip_path, analysis_name) + if not data: return None + loaded_data = joblib.load(BytesIO(data)) + + logger.info(f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}") + return loaded_data + else: # Load from separate pickle file if backtest_dir.is_dir(): From 0682d12fe38d3484f86591121e8d79c889e2dc8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Dec 2024 17:08:05 +0100 Subject: [PATCH 19/21] feat: get_backtest_market_change should support zip files --- freqtrade/data/btanalysis.py | 6 +++++- freqtrade/rpc/api_server/api_backtest.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 49b4b0eb1..d1884d00c 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -279,7 +279,11 @@ def get_backtest_market_change(filename: Path, include_ts: bool = True) -> pd.Da """ Read backtest market change file. """ - df = pd.read_feather(filename) + if filename.suffix == ".zip": + data = load_file_from_zip(filename, f"{filename.stem}_market_change.feather") + df = pd.read_feather(BytesIO(data)) + else: + df = pd.read_feather(filename) if include_ts: df.loc[:, "__date_ts"] = df.loc[:, "date"].astype(np.int64) // 1000 // 1000 return df diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 088a0a700..7346b22e0 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -350,10 +350,17 @@ def api_update_backtest_history_entry( ) def api_get_backtest_market_change(file: str, config=Depends(get_config)): bt_results_base: Path = config["user_data_dir"] / "backtest_results" - file_abs = (bt_results_base / f"{file}_market_change").with_suffix(".feather") - # Ensure file is in backtest_results directory - if not is_file_in_dir(file_abs, bt_results_base): + for fn in ( + Path(file).with_suffix(".zip"), + Path(f"{file}_market_change").with_suffix(".feather"), + ): + file_abs = bt_results_base / fn + # Ensure file is in backtest_results directory + if is_file_in_dir(file_abs, bt_results_base): + break + else: raise HTTPException(status_code=404, detail="File not found.") + df = get_backtest_market_change(file_abs) return { From 7a619594f6ef037e51d9532a06491fe689964d6a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2024 15:14:57 +0100 Subject: [PATCH 20/21] chore: improve zip file loading interface --- freqtrade/data/btanalysis.py | 5 +++-- freqtrade/optimize/optimize_reports/bt_storage.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index d1884d00c..fe85b08ea 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -404,12 +404,13 @@ def load_backtest_data(filename: Path | str, strategy: str | None = None) -> pd. return df -def load_file_from_zip(zip_path: Path, filename: str) -> bytes | None: +def load_file_from_zip(zip_path: Path, filename: str) -> bytes: """ Load a file from a zip file :param zip_path: Path to the zip file :param filename: Name of the file to load :return: Bytes of the file + :raises: ValueError if loading goes wrong. """ try: with zipfile.ZipFile(zip_path) as zipf: @@ -417,7 +418,7 @@ def load_file_from_zip(zip_path: Path, filename: str) -> bytes | None: return file.read() except zipfile.BadZipFile: logger.exception(f"Bad zip file: {zip_path}") - return None + raise ValueError(f"Bad zip file: {zip_path}") from None def load_backtest_analysis_data(backtest_dir: Path, name: str): diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py index b6b110638..d0c5d7fb7 100644 --- a/freqtrade/optimize/optimize_reports/bt_storage.py +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -1,7 +1,7 @@ import logging from io import BytesIO, StringIO from pathlib import Path -from typing import Any, TextIO +from typing import Any from zipfile import ZIP_DEFLATED, ZipFile from pandas import DataFrame @@ -16,7 +16,7 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename logger = logging.getLogger(__name__) -def file_dump_joblib(file_obj: TextIO, data: Any, log: bool = True) -> None: +def file_dump_joblib(file_obj: BytesIO, data: Any, log: bool = True) -> None: """ Dump object data into a file :param filename: file to create From 6c94b75172850b45654162e2eca3f20271350300 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Dec 2024 15:22:23 +0100 Subject: [PATCH 21/21] chore: improve zip loading errorhandling --- freqtrade/data/btanalysis.py | 14 ++++++++++---- tests/data/test_btanalysis.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index fe85b08ea..7b0109fa9 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -414,11 +414,17 @@ def load_file_from_zip(zip_path: Path, filename: str) -> bytes: """ try: with zipfile.ZipFile(zip_path) as zipf: - with zipf.open(filename) as file: - return file.read() + try: + with zipf.open(filename) as file: + return file.read() + except KeyError: + logger.error(f"File {filename} not found in zip: {zip_path}") + raise ValueError(f"File {filename} not found in zip: {zip_path}") from None + except FileNotFoundError: + raise ValueError(f"Zip file {zip_path} not found.") except zipfile.BadZipFile: - logger.exception(f"Bad zip file: {zip_path}") - raise ValueError(f"Bad zip file: {zip_path}") from None + logger.error(f"Bad zip file: {zip_path}.") + raise ValueError(f"Bad zip file: {zip_path}.") from None def load_backtest_analysis_data(backtest_dir: Path, name: str): diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 9b79983af..1c901bc16 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock +from zipfile import ZipFile import pytest from pandas import DataFrame, DateOffset, Timestamp, to_datetime @@ -15,6 +16,7 @@ from freqtrade.data.btanalysis import ( get_latest_hyperopt_file, load_backtest_data, load_backtest_metadata, + load_file_from_zip, load_trades, load_trades_from_db, ) @@ -569,3 +571,22 @@ def test_calculate_max_drawdown_abs(profits, relative, highd, lowdays, result, r assert drawdown.high_value > drawdown.low_value assert drawdown.drawdown_abs == result assert pytest.approx(drawdown.relative_account_drawdown) == result_rel + + +def test_load_file_from_zip(tmp_path): + with pytest.raises(ValueError, match=r"Zip file .* not found\."): + load_file_from_zip(tmp_path / "test.zip", "testfile.txt") + + (tmp_path / "testfile.zip").touch() + with pytest.raises(ValueError, match=r"Bad zip file.*"): + load_file_from_zip(tmp_path / "testfile.zip", "testfile.txt") + + zip_file = tmp_path / "testfile2.zip" + with ZipFile(zip_file, "w") as zipf: + zipf.writestr("testfile.txt", "testfile content") + + content = load_file_from_zip(zip_file, "testfile.txt") + assert content.decode("utf-8") == "testfile content" + + with pytest.raises(ValueError, match=r"File .* not found in zip.*"): + load_file_from_zip(zip_file, "testfile55.txt")