From b51c937e87a483154bd4a15856b17222a5cc30cf Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Tue, 27 May 2025 13:38:03 +0200 Subject: [PATCH 1/6] fix hyperopt repeated parameters between batches --- freqtrade/optimize/hyperopt/hyperopt.py | 52 ++++++++----------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index 926aaaf8c..b270214e1 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -15,6 +15,7 @@ from typing import Any import rapidjson from joblib import Parallel, cpu_count +from optuna.trial import Trial, TrialState from freqtrade.constants import FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.enums import HyperoptState @@ -169,8 +170,20 @@ class Hyperopt: asked.append(self.opt.ask(dimensions)) return asked + def check_optuna_asked_points(self, trial: Trial) -> bool: + trials_to_consider = trial.study.get_trials(deepcopy=False, states=[TrialState.COMPLETE]) + # Check whether we already evaluated the sampled `params`. + for t in reversed(trials_to_consider): + if trial.params == t.params: + logger.warning( + f"duplicate trial: Trial {trial.number} has same params as {t.number}" + ) + return True + return False + def get_asked_points(self, n_points: int, dimensions: dict) -> tuple[list[Any], list[bool]]: """ + TBD: need to change Enforce points returned from `self.opt.ask` have not been already evaluated Steps: @@ -181,44 +194,11 @@ class Hyperopt: 5. Repeat until at least `n_points` points in the `asked_non_tried` list 6. Return a list with length truncated at `n_points` """ - - def unique_list(a_list): - new_list = [] - for item in a_list: - if item not in new_list: - new_list.append(item) - return new_list - - i = 0 asked_non_tried: list[list[Any]] = [] - is_random_non_tried: list[bool] = [] - while i < 5 and len(asked_non_tried) < n_points: - if i < 3: - self.opt.cache_ = {} - asked = unique_list( - self.get_optuna_asked_points( - n_points=n_points * 5 if i > 0 else n_points, dimensions=dimensions - ) - ) - is_random = [False for _ in range(len(asked))] - else: - asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5)) - is_random = [True for _ in range(len(asked))] - is_random_non_tried += [ - rand for x, rand in zip(asked, is_random, strict=False) if x not in asked_non_tried - ] - asked_non_tried += [x for x in asked if x not in asked_non_tried] - i += 1 + optuna_asked_trials = self.get_optuna_asked_points(n_points=n_points, dimensions=dimensions) + asked_non_tried += [x for x in optuna_asked_trials if not self.check_optuna_asked_points(x)] - if asked_non_tried: - return ( - asked_non_tried[: min(len(asked_non_tried), n_points)], - is_random_non_tried[: min(len(asked_non_tried), n_points)], - ) - else: - return self.get_optuna_asked_points(n_points=n_points, dimensions=dimensions), [ - False for _ in range(n_points) - ] + return asked_non_tried, [False for _ in range(n_points)] def evaluate_result(self, val: dict[str, Any], current: int, is_random: bool): """ From 53383f31849da7893cba85f44dc85085ac4b13e2 Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Wed, 28 May 2025 09:35:20 +0200 Subject: [PATCH 2/6] add up to 5 retries for ask in case of duplicate params --- freqtrade/optimize/hyperopt/hyperopt.py | 41 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index b270214e1..a1cc0b808 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -92,6 +92,7 @@ class Hyperopt: self.print_json = self.config.get("print_json", False) self.hyperopter = HyperOptimizer(self.config, self.data_pickle_file) + self.count_skipped_epochs = 0 @staticmethod def get_lock_filename(config: Config) -> str: @@ -170,35 +171,46 @@ class Hyperopt: asked.append(self.opt.ask(dimensions)) return asked - def check_optuna_asked_points(self, trial: Trial) -> bool: + def duplicate_optuna_asked_points(self, trial: Trial) -> bool: trials_to_consider = trial.study.get_trials(deepcopy=False, states=[TrialState.COMPLETE]) # Check whether we already evaluated the sampled `params`. for t in reversed(trials_to_consider): if trial.params == t.params: - logger.warning( - f"duplicate trial: Trial {trial.number} has same params as {t.number}" - ) + # logger.warning( + # f"duplicate trial: Trial {trial.number} has same params as {t.number}" + # ) return True return False def get_asked_points(self, n_points: int, dimensions: dict) -> tuple[list[Any], list[bool]]: """ - TBD: need to change Enforce points returned from `self.opt.ask` have not been already evaluated Steps: 1. Try to get points using `self.opt.ask` first 2. Discard the points that have already been evaluated - 3. Retry using `self.opt.ask` up to 3 times - 4. If still some points are missing in respect to `n_points`, random sample some points - 5. Repeat until at least `n_points` points in the `asked_non_tried` list - 6. Return a list with length truncated at `n_points` + 3. Retry using `self.opt.ask` up to 5 times """ asked_non_tried: list[list[Any]] = [] + asked_duplicates: list[Trial] = [] optuna_asked_trials = self.get_optuna_asked_points(n_points=n_points, dimensions=dimensions) - asked_non_tried += [x for x in optuna_asked_trials if not self.check_optuna_asked_points(x)] + asked_non_tried += [ + x for x in optuna_asked_trials if not self.duplicate_optuna_asked_points(x) + ] + i = 0 + while i < 5 and len(asked_non_tried) < n_points: + asked_new = self.get_optuna_asked_points(n_points=1, dimensions=dimensions)[0] + if not self.duplicate_optuna_asked_points(asked_new): + asked_non_tried.append(asked_new) + else: + asked_duplicates.append(asked_new) + i += 1 + if len(asked_duplicates) > 0 and len(asked_non_tried) < n_points: + for asked_duplicate in asked_duplicates: + logger.warning(f"duplicate params for Epoch {asked_duplicate.number}") + self.count_skipped_epochs += len(asked_duplicates) - return asked_non_tried, [False for _ in range(n_points)] + return asked_non_tried, [False for _ in range(len(asked_non_tried))] def evaluate_result(self, val: dict[str, Any], current: int, is_random: bool): """ @@ -284,6 +296,7 @@ class Hyperopt: parallel, [asked1.params for asked1 in asked], ) + f_val_loss = [v["loss"] for v in f_val] for o_ask, v in zip(asked, f_val_loss, strict=False): self.opt.tell(o_ask, v) @@ -307,6 +320,12 @@ class Hyperopt: except KeyboardInterrupt: print("User interrupted..") + if self.count_skipped_epochs > 0: + logger.info( + f"{self.count_skipped_epochs} {plural(self.count_skipped_epochs, 'epoch')} " + f"skipped due to duplicate parameters." + ) + logger.info( f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " f"saved to '{self.results_file}'." From dfae7ca2ecdb4677e3ceb25a76288a4abb1364d3 Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Thu, 29 May 2025 15:41:47 +0300 Subject: [PATCH 3/6] fix duplicate params in same batch also --- freqtrade/optimize/hyperopt/hyperopt.py | 38 ++++++++++++++----------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index a1cc0b808..f00cdde78 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -15,7 +15,7 @@ from typing import Any import rapidjson from joblib import Parallel, cpu_count -from optuna.trial import Trial, TrialState +from optuna.trial import FrozenTrial, Trial, TrialState from freqtrade.constants import FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.enums import HyperoptState @@ -171,15 +171,19 @@ class Hyperopt: asked.append(self.opt.ask(dimensions)) return asked - def duplicate_optuna_asked_points(self, trial: Trial) -> bool: + def duplicate_optuna_asked_points(self, trial: Trial, asked_trials: list[FrozenTrial]) -> bool: + asked_trials_no_dups: list[FrozenTrial] = [] trials_to_consider = trial.study.get_trials(deepcopy=False, states=[TrialState.COMPLETE]) # Check whether we already evaluated the sampled `params`. for t in reversed(trials_to_consider): if trial.params == t.params: - # logger.warning( - # f"duplicate trial: Trial {trial.number} has same params as {t.number}" - # ) return True + # Check whether same`params` in one batch (asked_trials). Autosampler is doing this. + for t in asked_trials: + if t.params not in asked_trials_no_dups: + asked_trials_no_dups.append(t) + if len(asked_trials_no_dups) != len(asked_trials): + return True return False def get_asked_points(self, n_points: int, dimensions: dict) -> tuple[list[Any], list[bool]]: @@ -189,26 +193,26 @@ class Hyperopt: Steps: 1. Try to get points using `self.opt.ask` first 2. Discard the points that have already been evaluated - 3. Retry using `self.opt.ask` up to 5 times + 3. Retry using `self.opt.ask` up to `n_points` times """ - asked_non_tried: list[list[Any]] = [] - asked_duplicates: list[Trial] = [] + asked_non_tried: list[FrozenTrial] = [] optuna_asked_trials = self.get_optuna_asked_points(n_points=n_points, dimensions=dimensions) asked_non_tried += [ - x for x in optuna_asked_trials if not self.duplicate_optuna_asked_points(x) + x + for x in optuna_asked_trials + if not self.duplicate_optuna_asked_points(x, optuna_asked_trials) ] i = 0 - while i < 5 and len(asked_non_tried) < n_points: + while i < 2 * n_points and len(asked_non_tried) < n_points: asked_new = self.get_optuna_asked_points(n_points=1, dimensions=dimensions)[0] - if not self.duplicate_optuna_asked_points(asked_new): + if not self.duplicate_optuna_asked_points(asked_new, asked_non_tried): asked_non_tried.append(asked_new) - else: - asked_duplicates.append(asked_new) i += 1 - if len(asked_duplicates) > 0 and len(asked_non_tried) < n_points: - for asked_duplicate in asked_duplicates: - logger.warning(f"duplicate params for Epoch {asked_duplicate.number}") - self.count_skipped_epochs += len(asked_duplicates) + if len(asked_non_tried) < n_points: + logger.warning( + "duplicate params detected. Please check if search space is not too small!" + ) + self.count_skipped_epochs += n_points - len(asked_non_tried) return asked_non_tried, [False for _ in range(len(asked_non_tried))] From 14cfdb13c5fd74a9aea790ca95ade7388010e443 Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Fri, 30 May 2025 18:17:15 +0300 Subject: [PATCH 4/6] add back INITIAL_POINTS (default 30) for Samplers which support this --- freqtrade/optimize/hyperopt/hyperopt_optimizer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py index d0346f23a..94be5e50e 100644 --- a/freqtrade/optimize/hyperopt/hyperopt_optimizer.py +++ b/freqtrade/optimize/hyperopt/hyperopt_optimizer.py @@ -45,6 +45,7 @@ from freqtrade.util.dry_run_wallet import get_dry_run_wallet logger = logging.getLogger(__name__) +INITIAL_POINTS = 30 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -425,7 +426,16 @@ class HyperOptimizer: raise OperationalException(f"Optuna Sampler {o_sampler} not supported.") with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=ExperimentalWarning) - sampler = optuna_samplers_dict[o_sampler](seed=random_state) + if o_sampler in ["NSGAIIISampler", "NSGAIISampler"]: + sampler = optuna_samplers_dict[o_sampler]( + seed=random_state, population_size=INITIAL_POINTS + ) + elif o_sampler in ["GPSampler", "TPESampler", "CmaEsSampler"]: + sampler = optuna_samplers_dict[o_sampler]( + seed=random_state, n_startup_trials=INITIAL_POINTS + ) + else: + sampler = optuna_samplers_dict[o_sampler](seed=random_state) else: sampler = o_sampler From 12d31c4acb43a55a5518960058dbf05700ce5f4a Mon Sep 17 00:00:00 2001 From: viotemp1 Date: Sat, 31 May 2025 08:21:44 +0300 Subject: [PATCH 5/6] keep INITIAL_POINTS only in hyperopt_optimizer.py --- freqtrade/optimize/hyperopt/hyperopt.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index f00cdde78..db4567c39 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -22,7 +22,7 @@ from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import file_dump_json, plural from freqtrade.optimize.hyperopt.hyperopt_logger import logging_mp_handle, logging_mp_setup -from freqtrade.optimize.hyperopt.hyperopt_optimizer import HyperOptimizer +from freqtrade.optimize.hyperopt.hyperopt_optimizer import INITIAL_POINTS, HyperOptimizer from freqtrade.optimize.hyperopt.hyperopt_output import HyperoptOutput from freqtrade.optimize.hyperopt_tools import ( HyperoptStateContainer, @@ -35,9 +35,6 @@ from freqtrade.util import get_progress_tracker logger = logging.getLogger(__name__) -INITIAL_POINTS = 30 - - log_queue: Any From ae9073885f79a02cfc5721fb848d42f233658dc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 31 May 2025 15:55:34 +0200 Subject: [PATCH 6/6] chore: Update log wording, only log "duplicate parameters" once --- freqtrade/optimize/hyperopt/hyperopt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt/hyperopt.py b/freqtrade/optimize/hyperopt/hyperopt.py index db4567c39..9c2fa2016 100644 --- a/freqtrade/optimize/hyperopt/hyperopt.py +++ b/freqtrade/optimize/hyperopt/hyperopt.py @@ -206,9 +206,8 @@ class Hyperopt: asked_non_tried.append(asked_new) i += 1 if len(asked_non_tried) < n_points: - logger.warning( - "duplicate params detected. Please check if search space is not too small!" - ) + if self.count_skipped_epochs == 0: + logger.warning("Duplicate params detected. Maybe your search space is too small?") self.count_skipped_epochs += n_points - len(asked_non_tried) return asked_non_tried, [False for _ in range(len(asked_non_tried))]