From 90aaaa50fcce5808935299f8f89d7f3bb83f56a0 Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Wed, 2 Apr 2025 18:45:49 +0300 Subject: [PATCH] fix increasing memory usage. --- freqtrade/optimize/hyperopt/hyperopt.py | 33 ++++++--- .../optimize/hyperopt/hyperopt_optimizer.py | 69 ++++++++++--------- tests/optimize/test_hyperopt.py | 4 +- 3 files changed, 65 insertions(+), 41 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index 4dc720475..5c906cb9a 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -4,6 +4,7 @@ This module contains the hyperopt logic """ +import gc import logging import random from datetime import datetime @@ -19,6 +20,7 @@ from freqtrade.constants import FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import file_dump_json, plural +from freqtrade.optimize.backtesting import Backtesting from freqtrade.optimize.hyperopt.hyperopt_logger import logging_mp_handle, logging_mp_setup from freqtrade.optimize.hyperopt.hyperopt_optimizer import HyperOptimizer from freqtrade.optimize.hyperopt.hyperopt_output import HyperoptOutput @@ -30,6 +32,9 @@ from freqtrade.optimize.hyperopt_tools import ( from freqtrade.util import get_progress_tracker +# import multiprocessing as mp +# mp.set_start_method('fork', force=True) # spawn fork forkserver + logger = logging.getLogger(__name__) @@ -90,6 +95,7 @@ class Hyperopt: self.print_json = self.config.get("print_json", False) self.hyperopter = HyperOptimizer(self.config) + self.hyperopter.data_pickle_file = self.data_pickle_file @staticmethod def get_lock_filename(config: Config) -> str: @@ -146,7 +152,9 @@ class Hyperopt: self.print_all, ) - def run_optimizer_parallel(self, parallel: Parallel, asked: list[list]) -> list[dict[str, Any]]: + def run_optimizer_parallel( + self, parallel: Parallel, backtesting: Backtesting, asked: list[list] + ) -> list[dict[str, Any]]: """Start optimizer in a parallel way""" def optimizer_wrapper(*args, **kwargs): @@ -157,7 +165,9 @@ class Hyperopt: return self.hyperopter.generate_optimizer(*args, **kwargs) - return parallel(delayed(wrap_non_picklable_objects(optimizer_wrapper))(v) for v in asked) + return parallel( + delayed(wrap_non_picklable_objects(optimizer_wrapper))(backtesting, v) for v in asked + ) def _set_random_state(self, random_state: int | None) -> int: return random_state or random.randint(1, 2**16 - 1) # noqa: S311 @@ -282,7 +292,9 @@ class Hyperopt: asked, is_random = self.get_asked_points( n_points=1, dimensions=self.hyperopter.o_dimensions ) - f_val0 = self.hyperopter.generate_optimizer(asked[0].params) + f_val0 = self.hyperopter.generate_optimizer( + self.hyperopter.backtesting, asked[0].params + ) self.opt.tell(asked[0], [f_val0["loss"]]) self.evaluate_result(f_val0, 1, is_random[0]) pbar.update(task, advance=1) @@ -299,13 +311,17 @@ class Hyperopt: n_points=current_jobs, dimensions=self.hyperopter.o_dimensions ) # asked_params = [asked1.params for asked1 in asked] - # logger.info(f"asked iteration {i}: {asked_params}") + # logger.info(f"asked iteration {i}: {asked} {asked_params}") + f_val = self.run_optimizer_parallel( - parallel, [asked1.params for asked1 in asked] + parallel, + self.hyperopter.backtesting, + [asked1.params for asked1 in asked], ) - for o_ask, v in zip(asked, f_val, strict=False): - self.opt.tell(o_ask, v["loss"]) - # self.opt.tell(asked, [v["loss"] for v in f_val]) + f_val_loss = [v["loss"] for v in f_val] + for o_ask, v in zip(asked, f_val_loss, strict=False): + self.opt.tell(o_ask, v) + # logger.info(f"result iteration {i}: {asked} {f_val_loss}") for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) @@ -314,6 +330,7 @@ class Hyperopt: self.evaluate_result(val, current, is_random[j]) pbar.update(task, advance=1) logging_mp_handle(log_queue) + gc.collect() except KeyboardInterrupt: print("User interrupted..") diff --git a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py index af42f486d..19a30015e 100644 --- a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py +++ b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py @@ -7,12 +7,14 @@ import logging import sys import warnings from datetime import datetime, timezone +from pathlib import Path from typing import Any from joblib import dump, load from joblib.externals import cloudpickle from pandas import DataFrame +# from memory_profiler import profile from freqtrade.constants import DATETIME_PRINT_FORMAT, Config from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange @@ -31,10 +33,12 @@ from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver from freqtrade.util.dry_run_wallet import get_dry_run_wallet -# Suppress scikit-learn FutureWarnings from skopt +# Suppress optuna ExperimentalWarning from skopt with warnings.catch_warnings(): + from optuna.exceptions import ExperimentalWarning + warnings.filterwarnings("ignore", category=FutureWarning) - # from skopt import Optimizer + # warnings.filterwarnings("ignore", category=ExperimentalWarning) import optuna from skopt.space import Dimension @@ -102,12 +106,8 @@ class HyperOptimizer: self.config ) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - - self.data_pickle_file = ( - self.config["user_data_dir"] / "hyperopt_results" / "hyperopt_tickerdata.pkl" - ) - self.market_change = 0.0 + self.data_pickle_file = "" if HyperoptTools.has_space(self.config, "sell"): # Make sure use_exit_signal is enabled @@ -260,16 +260,23 @@ class HyperOptimizer: + self.max_open_trades_space ) - def assign_params(self, params_dict: dict[str, Any], category: str) -> None: + def assign_params( + self, backtesting: Backtesting, params_dict: dict[str, Any], category: str + ) -> None: """ Assign hyperoptable parameters """ - for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category): + for attr_name, attr in backtesting.strategy.enumerate_parameters(category): if attr.optimize: # noinspection PyProtectedMember attr.value = params_dict[attr_name] - def generate_optimizer(self, raw_params: dict[str, Any]) -> dict[str, Any]: # list[Any] + # @profile + # fp=open('memory_profiler.log','w+') + # @profile(stream=fp) + def generate_optimizer( + self, backtesting: Backtesting, raw_params: dict[str, Any] + ) -> dict[str, Any]: # list[Any] """ Used Optimize function. Called once per epoch to optimize whatever is configured. @@ -281,30 +288,26 @@ class HyperOptimizer: # Apply parameters if HyperoptTools.has_space(self.config, "buy"): - self.assign_params(params_dict, "buy") + self.assign_params(backtesting, params_dict, "buy") if HyperoptTools.has_space(self.config, "sell"): - self.assign_params(params_dict, "sell") + self.assign_params(backtesting, params_dict, "sell") if HyperoptTools.has_space(self.config, "protection"): - self.assign_params(params_dict, "protection") + self.assign_params(backtesting, params_dict, "protection") if HyperoptTools.has_space(self.config, "roi"): - self.backtesting.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table( - params_dict - ) + backtesting.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params_dict) if HyperoptTools.has_space(self.config, "stoploss"): - self.backtesting.strategy.stoploss = params_dict["stoploss"] + backtesting.strategy.stoploss = params_dict["stoploss"] if HyperoptTools.has_space(self.config, "trailing"): d = self.custom_hyperopt.generate_trailing_params(params_dict) - self.backtesting.strategy.trailing_stop = d["trailing_stop"] - self.backtesting.strategy.trailing_stop_positive = d["trailing_stop_positive"] - self.backtesting.strategy.trailing_stop_positive_offset = d[ - "trailing_stop_positive_offset" - ] - self.backtesting.strategy.trailing_only_offset_is_reached = d[ + backtesting.strategy.trailing_stop = d["trailing_stop"] + backtesting.strategy.trailing_stop_positive = d["trailing_stop_positive"] + backtesting.strategy.trailing_stop_positive_offset = d["trailing_stop_positive_offset"] + backtesting.strategy.trailing_only_offset_is_reached = d[ "trailing_only_offset_is_reached" ] @@ -323,15 +326,15 @@ class HyperOptimizer: self.config.update({"max_open_trades": updated_max_open_trades}) - self.backtesting.strategy.max_open_trades = updated_max_open_trades + backtesting.strategy.max_open_trades = updated_max_open_trades - with self.data_pickle_file.open("rb") as f: + with Path(self.data_pickle_file).open("rb") as f: processed = load(f, mmap_mode="r") - if self.analyze_per_epoch: - # Data is not yet analyzed, rerun populate_indicators. - processed = self.advise_and_trim(processed) + if self.analyze_per_epoch: + # Data is not yet analyzed, rerun populate_indicators. + processed = self.advise_and_trim(processed) - bt_results = self.backtesting.backtest( + bt_results = backtesting.backtest( processed=processed, start_date=self.min_date, end_date=self.max_date ) backtest_end_time = datetime.now(timezone.utc) @@ -341,10 +344,10 @@ class HyperOptimizer: "backtest_end_time": int(backtest_end_time.timestamp()), } ) - - return self._get_results_dict( + result = self._get_results_dict( bt_results, self.min_date, self.max_date, params_dict, processed=processed ) + return result def _get_results_dict( self, @@ -443,7 +446,9 @@ class HyperOptimizer: if isinstance(o_sampler, str): if o_sampler not in optuna_samplers_dict.keys(): raise OperationalException(f"Optuna Sampler {o_sampler} not supported.") - sampler = optuna_samplers_dict[o_sampler](seed=random_state) + with warnings.catch_warnings(): + warnings.filterwarnings(action="ignore", category=ExperimentalWarning) + sampler = optuna_samplers_dict[o_sampler](seed=random_state) else: sampler = o_sampler diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index ad684dd19..e10077f7f 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -608,7 +608,9 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: hyperopt.hyperopter.min_date = dt_utc(2017, 12, 10) hyperopt.hyperopter.max_date = dt_utc(2017, 12, 13) hyperopt.hyperopter.init_spaces() - generate_optimizer_value = hyperopt.hyperopter.generate_optimizer(optimizer_param) + generate_optimizer_value = hyperopt.hyperopter.generate_optimizer( + hyperopt.hyperopter.backtesting, optimizer_param + ) # list(optimizer_param.values()) assert generate_optimizer_value == response_expected