From da7addcd98ddac0132d38369a42da4441f8bb69f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 May 2024 17:16:02 +0200 Subject: [PATCH] ruff format: hyperopt --- freqtrade/optimize/hyperopt.py | 391 ++++++++------- freqtrade/optimize/hyperopt_auto.py | 53 ++- freqtrade/optimize/hyperopt_epoch_filters.py | 103 ++-- freqtrade/optimize/hyperopt_interface.py | 113 ++--- freqtrade/optimize/hyperopt_loss_interface.py | 17 +- freqtrade/optimize/hyperopt_tools.py | 449 +++++++++++------- 6 files changed, 630 insertions(+), 496 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3725d1f16..6c54df72f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -93,30 +93,36 @@ class Hyperopt: self.backtesting = Backtesting(self.config) self.pairlist = self.backtesting.pairlists.whitelist self.custom_hyperopt: HyperOptAuto - self.analyze_per_epoch = self.config.get('analyze_per_epoch', False) + self.analyze_per_epoch = self.config.get("analyze_per_epoch", False) HyperoptStateContainer.set_state(HyperoptState.STARTUP) - if not self.config.get('hyperopt'): + if not self.config.get("hyperopt"): self.custom_hyperopt = HyperOptAuto(self.config) else: raise OperationalException( "Using separate Hyperopt files has been removed in 2021.9. Please convert " - "your existing Hyperopt file to the new Hyperoptable strategy interface") + "your existing Hyperopt file to the new Hyperoptable strategy interface" + ) self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy self.hyperopt_pickle_magic(self.backtesting.strategy.__class__.__bases__) self.custom_hyperoptloss: IHyperOptLoss = HyperOptLossResolver.load_hyperoptloss( - self.config) + self.config + ) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - strategy = str(self.config['strategy']) - self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' / - f'strategy_{strategy}_{time_now}.fthypt') - self.data_pickle_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_tickerdata.pkl') - self.total_epochs = config.get('epochs', 0) + strategy = str(self.config["strategy"]) + self.results_file: Path = ( + self.config["user_data_dir"] + / "hyperopt_results" + / f"strategy_{strategy}_{time_now}.fthypt" + ) + self.data_pickle_file = ( + self.config["user_data_dir"] / "hyperopt_results" / "hyperopt_tickerdata.pkl" + ) + self.total_epochs = config.get("epochs", 0) self.current_best_loss = 100 @@ -127,24 +133,23 @@ class Hyperopt: self.current_best_epoch: Optional[Dict[str, Any]] = None # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set - if not self.config.get('use_max_market_positions', True): - logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - self.backtesting.strategy.max_open_trades = float('inf') - config.update({'max_open_trades': self.backtesting.strategy.max_open_trades}) + if not self.config.get("use_max_market_positions", True): + logger.debug("Ignoring max_open_trades (--disable-max-market-positions was used) ...") + self.backtesting.strategy.max_open_trades = float("inf") + config.update({"max_open_trades": self.backtesting.strategy.max_open_trades}) - if HyperoptTools.has_space(self.config, 'sell'): + if HyperoptTools.has_space(self.config, "sell"): # Make sure use_exit_signal is enabled - self.config['use_exit_signal'] = True + self.config["use_exit_signal"] = True - self.print_all = self.config.get('print_all', False) + self.print_all = self.config.get("print_all", False) self.hyperopt_table_header = 0 - self.print_colorized = self.config.get('print_colorized', False) - self.print_json = self.config.get('print_json', False) + self.print_colorized = self.config.get("print_colorized", False) + self.print_json = self.config.get("print_json", False) @staticmethod def get_lock_filename(config: Config) -> str: - - return str(config['user_data_dir'] / 'hyperopt.lock') + return str(config["user_data_dir"] / "hyperopt.lock") def clean_hyperopt(self) -> None: """ @@ -163,16 +168,15 @@ class Hyperopt: to pickle as value. """ for modules in bases: - if modules.__name__ != 'IStrategy': + if modules.__name__ != "IStrategy": cloudpickle.register_pickle_by_value(sys.modules[modules.__module__]) self.hyperopt_pickle_magic(modules.__bases__) def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict: - # Ensure the number of dimensions match # the number of parameters in the list. if len(raw_params) != len(dimensions): - raise ValueError('Mismatch in number of search-space dimensions.') + raise ValueError("Mismatch in number of search-space dimensions.") # Return a dict where the keys are the names of the dimensions # and the values are taken from the list of parameters. @@ -186,18 +190,23 @@ class Hyperopt: :param epoch: result dictionary for this epoch. """ epoch[FTHYPT_FILEVERSION] = 2 - with self.results_file.open('a') as f: - rapidjson.dump(epoch, f, default=hyperopt_serializer, - number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) + with self.results_file.open("a") as f: + rapidjson.dump( + epoch, + f, + default=hyperopt_serializer, + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN, + ) f.write("\n") self.num_epochs_saved += 1 - logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " - f"saved to '{self.results_file}'.") + logger.debug( + f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'." + ) # Store hyperopt filename latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN) - file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)}, - log=False) + file_dump_json(latest_filename, {"latest_hyperopt": str(self.results_file.name)}, log=False) def _get_params_details(self, params: Dict) -> Dict: """ @@ -205,23 +214,26 @@ class Hyperopt: """ result: Dict = {} - if HyperoptTools.has_space(self.config, 'buy'): - result['buy'] = {p.name: params.get(p.name) for p in self.buy_space} - if HyperoptTools.has_space(self.config, 'sell'): - result['sell'] = {p.name: params.get(p.name) for p in self.sell_space} - if HyperoptTools.has_space(self.config, 'protection'): - result['protection'] = {p.name: params.get(p.name) for p in self.protection_space} - if HyperoptTools.has_space(self.config, 'roi'): - result['roi'] = {str(k): v for k, v in - self.custom_hyperopt.generate_roi_table(params).items()} - if HyperoptTools.has_space(self.config, 'stoploss'): - result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space} - if HyperoptTools.has_space(self.config, 'trailing'): - result['trailing'] = self.custom_hyperopt.generate_trailing_params(params) - if HyperoptTools.has_space(self.config, 'trades'): - result['max_open_trades'] = { - 'max_open_trades': self.backtesting.strategy.max_open_trades - if self.backtesting.strategy.max_open_trades != float('inf') else -1} + if HyperoptTools.has_space(self.config, "buy"): + result["buy"] = {p.name: params.get(p.name) for p in self.buy_space} + if HyperoptTools.has_space(self.config, "sell"): + result["sell"] = {p.name: params.get(p.name) for p in self.sell_space} + if HyperoptTools.has_space(self.config, "protection"): + result["protection"] = {p.name: params.get(p.name) for p in self.protection_space} + if HyperoptTools.has_space(self.config, "roi"): + result["roi"] = { + str(k): v for k, v in self.custom_hyperopt.generate_roi_table(params).items() + } + if HyperoptTools.has_space(self.config, "stoploss"): + result["stoploss"] = {p.name: params.get(p.name) for p in self.stoploss_space} + if HyperoptTools.has_space(self.config, "trailing"): + result["trailing"] = self.custom_hyperopt.generate_trailing_params(params) + if HyperoptTools.has_space(self.config, "trades"): + result["max_open_trades"] = { + "max_open_trades": self.backtesting.strategy.max_open_trades + if self.backtesting.strategy.max_open_trades != float("inf") + else -1 + } return result @@ -231,19 +243,19 @@ class Hyperopt: """ result: Dict[str, Any] = {} strategy = self.backtesting.strategy - if not HyperoptTools.has_space(self.config, 'roi'): - result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()} - if not HyperoptTools.has_space(self.config, 'stoploss'): - result['stoploss'] = {'stoploss': strategy.stoploss} - if not HyperoptTools.has_space(self.config, 'trailing'): - result['trailing'] = { - 'trailing_stop': strategy.trailing_stop, - 'trailing_stop_positive': strategy.trailing_stop_positive, - 'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset, - 'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached, + if not HyperoptTools.has_space(self.config, "roi"): + result["roi"] = {str(k): v for k, v in strategy.minimal_roi.items()} + if not HyperoptTools.has_space(self.config, "stoploss"): + result["stoploss"] = {"stoploss": strategy.stoploss} + if not HyperoptTools.has_space(self.config, "trailing"): + result["trailing"] = { + "trailing_stop": strategy.trailing_stop, + "trailing_stop_positive": strategy.trailing_stop_positive, + "trailing_stop_positive_offset": strategy.trailing_stop_positive_offset, + "trailing_only_offset_is_reached": strategy.trailing_only_offset_is_reached, } - if not HyperoptTools.has_space(self.config, 'trades'): - result['max_open_trades'] = {'max_open_trades': strategy.max_open_trades} + if not HyperoptTools.has_space(self.config, "trades"): + result["max_open_trades"] = {"max_open_trades": strategy.max_open_trades} return result def print_results(self, results) -> None: @@ -251,14 +263,17 @@ class Hyperopt: Log results if it is better than any previous evaluation TODO: this should be moved to HyperoptTools too """ - is_best = results['is_best'] + is_best = results["is_best"] if self.print_all or is_best: print( HyperoptTools.get_result_table( - self.config, results, self.total_epochs, - self.print_all, self.print_colorized, - self.hyperopt_table_header + self.config, + results, + self.total_epochs, + self.print_all, + self.print_colorized, + self.hyperopt_table_header, ) ) self.hyperopt_table_header = 2 @@ -267,41 +282,47 @@ class Hyperopt: """ Assign the dimensions in the hyperoptimization space. """ - if HyperoptTools.has_space(self.config, 'protection'): + if HyperoptTools.has_space(self.config, "protection"): # Protections can only be optimized when using the Parameter interface logger.debug("Hyperopt has 'protection' space") # Enable Protections if protection space is selected. - self.config['enable_protections'] = True + self.config["enable_protections"] = True self.backtesting.enable_protections = True self.protection_space = self.custom_hyperopt.protection_space() - if HyperoptTools.has_space(self.config, 'buy'): + if HyperoptTools.has_space(self.config, "buy"): logger.debug("Hyperopt has 'buy' space") self.buy_space = self.custom_hyperopt.buy_indicator_space() - if HyperoptTools.has_space(self.config, 'sell'): + if HyperoptTools.has_space(self.config, "sell"): logger.debug("Hyperopt has 'sell' space") self.sell_space = self.custom_hyperopt.sell_indicator_space() - if HyperoptTools.has_space(self.config, 'roi'): + if HyperoptTools.has_space(self.config, "roi"): logger.debug("Hyperopt has 'roi' space") self.roi_space = self.custom_hyperopt.roi_space() - if HyperoptTools.has_space(self.config, 'stoploss'): + if HyperoptTools.has_space(self.config, "stoploss"): logger.debug("Hyperopt has 'stoploss' space") self.stoploss_space = self.custom_hyperopt.stoploss_space() - if HyperoptTools.has_space(self.config, 'trailing'): + if HyperoptTools.has_space(self.config, "trailing"): logger.debug("Hyperopt has 'trailing' space") self.trailing_space = self.custom_hyperopt.trailing_space() - if HyperoptTools.has_space(self.config, 'trades'): + if HyperoptTools.has_space(self.config, "trades"): logger.debug("Hyperopt has 'trades' space") self.max_open_trades_space = self.custom_hyperopt.max_open_trades_space() - self.dimensions = (self.buy_space + self.sell_space + self.protection_space - + self.roi_space + self.stoploss_space + self.trailing_space - + self.max_open_trades_space) + self.dimensions = ( + self.buy_space + + self.sell_space + + self.protection_space + + self.roi_space + + self.stoploss_space + + self.trailing_space + + self.max_open_trades_space + ) def assign_params(self, params_dict: Dict, category: str) -> None: """ @@ -323,104 +344,119 @@ class Hyperopt: params_dict = self._get_params_dict(self.dimensions, raw_params) # Apply parameters - if HyperoptTools.has_space(self.config, 'buy'): - self.assign_params(params_dict, 'buy') + if HyperoptTools.has_space(self.config, "buy"): + self.assign_params(params_dict, "buy") - if HyperoptTools.has_space(self.config, 'sell'): - self.assign_params(params_dict, 'sell') + if HyperoptTools.has_space(self.config, "sell"): + self.assign_params(params_dict, "sell") - if HyperoptTools.has_space(self.config, 'protection'): - self.assign_params(params_dict, 'protection') + if HyperoptTools.has_space(self.config, "protection"): + self.assign_params(params_dict, "protection") - if HyperoptTools.has_space(self.config, 'roi'): - self.backtesting.strategy.minimal_roi = ( - self.custom_hyperopt.generate_roi_table(params_dict)) + if HyperoptTools.has_space(self.config, "roi"): + self.backtesting.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table( + params_dict + ) - if HyperoptTools.has_space(self.config, 'stoploss'): - self.backtesting.strategy.stoploss = params_dict['stoploss'] + if HyperoptTools.has_space(self.config, "stoploss"): + self.backtesting.strategy.stoploss = params_dict["stoploss"] - if HyperoptTools.has_space(self.config, 'trailing'): + if HyperoptTools.has_space(self.config, "trailing"): d = self.custom_hyperopt.generate_trailing_params(params_dict) - self.backtesting.strategy.trailing_stop = d['trailing_stop'] - self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive'] - self.backtesting.strategy.trailing_stop_positive_offset = \ - d['trailing_stop_positive_offset'] - self.backtesting.strategy.trailing_only_offset_is_reached = \ - d['trailing_only_offset_is_reached'] + self.backtesting.strategy.trailing_stop = d["trailing_stop"] + self.backtesting.strategy.trailing_stop_positive = d["trailing_stop_positive"] + self.backtesting.strategy.trailing_stop_positive_offset = d[ + "trailing_stop_positive_offset" + ] + self.backtesting.strategy.trailing_only_offset_is_reached = d[ + "trailing_only_offset_is_reached" + ] - if HyperoptTools.has_space(self.config, 'trades'): - if self.config["stake_amount"] == "unlimited" and \ - (params_dict['max_open_trades'] == -1 or params_dict['max_open_trades'] == 0): + if HyperoptTools.has_space(self.config, "trades"): + if self.config["stake_amount"] == "unlimited" and ( + params_dict["max_open_trades"] == -1 or params_dict["max_open_trades"] == 0 + ): # Ignore unlimited max open trades if stake amount is unlimited - params_dict.update({'max_open_trades': self.config['max_open_trades']}) + params_dict.update({"max_open_trades": self.config["max_open_trades"]}) - updated_max_open_trades = int(params_dict['max_open_trades']) \ - if (params_dict['max_open_trades'] != -1 - and params_dict['max_open_trades'] != 0) else float('inf') + updated_max_open_trades = ( + int(params_dict["max_open_trades"]) + if (params_dict["max_open_trades"] != -1 and params_dict["max_open_trades"] != 0) + else float("inf") + ) - self.config.update({'max_open_trades': updated_max_open_trades}) + self.config.update({"max_open_trades": updated_max_open_trades}) self.backtesting.strategy.max_open_trades = updated_max_open_trades - with self.data_pickle_file.open('rb') as f: - processed = load(f, mmap_mode='r') + with self.data_pickle_file.open("rb") as f: + processed = load(f, mmap_mode="r") if self.analyze_per_epoch: # Data is not yet analyzed, rerun populate_indicators. processed = self.advise_and_trim(processed) bt_results = self.backtesting.backtest( - processed=processed, - start_date=self.min_date, - end_date=self.max_date + processed=processed, start_date=self.min_date, end_date=self.max_date ) backtest_end_time = datetime.now(timezone.utc) - bt_results.update({ - 'backtest_start_time': int(backtest_start_time.timestamp()), - 'backtest_end_time': int(backtest_end_time.timestamp()), - }) + bt_results.update( + { + "backtest_start_time": int(backtest_start_time.timestamp()), + "backtest_end_time": int(backtest_end_time.timestamp()), + } + ) - return self._get_results_dict(bt_results, self.min_date, self.max_date, - params_dict, - processed=processed) + return self._get_results_dict( + bt_results, self.min_date, self.max_date, params_dict, processed=processed + ) - def _get_results_dict(self, backtesting_results, min_date, max_date, - params_dict, processed: Dict[str, DataFrame] - ) -> Dict[str, Any]: + def _get_results_dict( + self, backtesting_results, min_date, max_date, params_dict, processed: Dict[str, DataFrame] + ) -> Dict[str, Any]: params_details = self._get_params_details(params_dict) strat_stats = generate_strategy_stats( - self.pairlist, self.backtesting.strategy.get_strategy_name(), - backtesting_results, min_date, max_date, market_change=self.market_change, + self.pairlist, + self.backtesting.strategy.get_strategy_name(), + backtesting_results, + min_date, + max_date, + market_change=self.market_change, is_hyperopt=True, ) results_explanation = HyperoptTools.format_results_explanation_string( - strat_stats, self.config['stake_currency']) + strat_stats, self.config["stake_currency"] + ) not_optimized = self.backtesting.strategy.get_no_optimize_params() not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details()) - trade_count = strat_stats['total_trades'] - total_profit = strat_stats['profit_total'] + trade_count = strat_stats["total_trades"] + total_profit = strat_stats["profit_total"] # If this evaluation contains too short amount of trades to be # interesting -- consider it as 'bad' (assigned max. loss value) # in order to cast this hyperspace point away from optimization # path. We do not want to optimize 'hodl' strategies. loss: float = MAX_LOSS - if trade_count >= self.config['hyperopt_min_trades']: - loss = self.calculate_loss(results=backtesting_results['results'], - trade_count=trade_count, - min_date=min_date, max_date=max_date, - config=self.config, processed=processed, - backtest_stats=strat_stats) + if trade_count >= self.config["hyperopt_min_trades"]: + loss = self.calculate_loss( + results=backtesting_results["results"], + trade_count=trade_count, + min_date=min_date, + max_date=max_date, + config=self.config, + processed=processed, + backtest_stats=strat_stats, + ) return { - 'loss': loss, - 'params_dict': params_dict, - 'params_details': params_details, - 'params_not_optimized': not_optimized, - 'results_metrics': strat_stats, - 'results_explanation': results_explanation, - 'total_profit': total_profit, + "loss": loss, + "params_dict": params_dict, + "params_details": params_details, + "params_not_optimized": not_optimized, + "results_metrics": strat_stats, + "results_explanation": results_explanation, + "total_profit": total_profit, } def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: @@ -439,16 +475,16 @@ class Hyperopt: base_estimator=estimator, acq_optimizer=acq_optimizer, n_initial_points=INITIAL_POINTS, - acq_optimizer_kwargs={'n_jobs': cpu_count}, + acq_optimizer_kwargs={"n_jobs": cpu_count}, random_state=self.random_state, model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def run_optimizer_parallel( - self, parallel: Parallel, asked: List[List]) -> List[Dict[str, Any]]: - """ Start optimizer in a parallel way """ - return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + def run_optimizer_parallel(self, parallel: Parallel, asked: List[List]) -> List[Dict[str, Any]]: + """Start optimizer in a parallel way""" + return parallel( + delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked + ) def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) @@ -462,7 +498,7 @@ class Hyperopt: trimmed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup) self.min_date, self.max_date = get_timerange(trimmed) if not self.market_change: - self.market_change = calculate_market_change(trimmed, 'close') + self.market_change = calculate_market_change(trimmed, "close") # Real trimming will happen as part of backtesting. return preprocessed @@ -478,10 +514,12 @@ class Hyperopt: preprocessed = self.advise_and_trim(data) - logger.info(f'Hyperopting with data from ' - f'{self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(self.max_date - self.min_date).days} days)..') + logger.info( + f"Hyperopting with data from " + f"{self.min_date.strftime(DATETIME_PRINT_FORMAT)} " + f"up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} " + f"({(self.max_date - self.min_date).days} days).." + ) # Store non-trimmed data - will be trimmed after signal generation. dump(preprocessed, self.data_pickle_file) else: @@ -499,12 +537,14 @@ class Hyperopt: 5. Repeat until at least `n_points` points in the `asked_non_tried` list 6. Return a list with length truncated at `n_points` """ + def unique_list(a_list): new_list = [] for item in a_list: if item not in new_list: new_list.append(item) return new_list + i = 0 asked_non_tried: List[List[Any]] = [] is_random_non_tried: List[bool] = [] @@ -516,18 +556,20 @@ class Hyperopt: else: asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5)) is_random = [True for _ in range(len(asked))] - is_random_non_tried += [rand for x, rand in zip(asked, is_random) - if x not in self.opt.Xi - and x not in asked_non_tried] - asked_non_tried += [x for x in asked - if x not in self.opt.Xi - and x not in asked_non_tried] + is_random_non_tried += [ + rand + for x, rand in zip(asked, is_random) + if x not in self.opt.Xi and x not in asked_non_tried + ] + asked_non_tried += [ + x for x in asked if x not in self.opt.Xi and x not in asked_non_tried + ] i += 1 if asked_non_tried: return ( - asked_non_tried[:min(len(asked_non_tried), n_points)], - is_random_non_tried[:min(len(asked_non_tried), n_points)] + asked_non_tried[: min(len(asked_non_tried), n_points)], + is_random_non_tried[: min(len(asked_non_tried), n_points)], ) else: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] @@ -536,8 +578,8 @@ class Hyperopt: """ Evaluate results returned from generate_optimizer """ - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS + val["current_epoch"] = current + val["is_initial_point"] = current <= INITIAL_POINTS logger.debug("Optimizer epoch evaluated: %s", val) @@ -546,18 +588,18 @@ class Hyperopt: # to keep proper order in the list of results. That's because # evaluations can take different time. Here they are aligned in the # order they will be shown to the user. - val['is_best'] = is_best - val['is_random'] = is_random + val["is_best"] = is_best + val["is_random"] = is_random self.print_results(val) if is_best: - self.current_best_loss = val['loss'] + self.current_best_loss = val["loss"] self.current_best_epoch = val self._save_result(val) def start(self) -> None: - self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) + self.random_state = self._set_random_state(self.config.get("hyperopt_random_state")) logger.info(f"Using optimizer random state: {self.random_state}") self.hyperopt_table_header = -1 # Initialize spaces ... @@ -577,8 +619,8 @@ class Hyperopt: cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") - config_jobs = self.config.get('hyperopt_jobs', -1) - logger.info(f'Number of parallel jobs set as: {config_jobs}') + config_jobs = self.config.get("hyperopt_jobs", -1) + logger.info(f"Number of parallel jobs set as: {config_jobs}") self.opt = self.get_optimizer(self.dimensions, config_jobs) @@ -588,7 +630,7 @@ class Hyperopt: try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() - logger.info(f'Effective number of parallel workers used: {jobs}') + logger.info(f"Effective number of parallel workers used: {jobs}") # Define progressbar with Progress( @@ -611,7 +653,7 @@ class Hyperopt: # This allows dataprovider to load it's informative cache. asked, is_random = self.get_asked_points(n_points=1) f_val0 = self.generate_optimizer(asked[0]) - self.opt.tell(asked, [f_val0['loss']]) + self.opt.tell(asked, [f_val0["loss"]]) self.evaluate_result(f_val0, 1, is_random[0]) pbar.update(task, advance=1) start += 1 @@ -625,7 +667,7 @@ class Hyperopt: asked, is_random = self.get_asked_points(n_points=current_jobs) f_val = self.run_optimizer_parallel(parallel, asked) - self.opt.tell(asked, [v['loss'] for v in f_val]) + self.opt.tell(asked, [v["loss"] for v in f_val]) for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) @@ -635,23 +677,26 @@ class Hyperopt: pbar.update(task, advance=1) except KeyboardInterrupt: - print('User interrupted..') + print("User interrupted..") - logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " - f"saved to '{self.results_file}'.") + logger.info( + f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'." + ) if self.current_best_epoch: HyperoptTools.try_export_params( - self.config, - self.backtesting.strategy.get_strategy_name(), - self.current_best_epoch) + self.config, self.backtesting.strategy.get_strategy_name(), self.current_best_epoch + ) - HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, - self.print_json) + HyperoptTools.show_epoch_details( + self.current_best_epoch, self.total_epochs, self.print_json + ) elif self.num_epochs_saved > 0: print( f"No good result found for given optimization function in {self.num_epochs_saved} " - f"{plural(self.num_epochs_saved, 'epoch')}.") + f"{plural(self.num_epochs_saved, 'epoch')}." + ) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index a3e2ef058..cf0103162 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -3,6 +3,7 @@ HyperOptAuto class. This module implements a convenience auto-hyperopt class, which can be used together with strategies that implement IHyperStrategy interface. """ + import logging from contextlib import suppress from typing import Callable, Dict, List @@ -20,15 +21,17 @@ logger = logging.getLogger(__name__) def _format_exception_message(space: str, ignore_missing_space: bool) -> None: - msg = (f"The '{space}' space is included into the hyperoptimization " - f"but no parameter for this space was found in your Strategy. " - ) + msg = ( + f"The '{space}' space is included into the hyperoptimization " + f"but no parameter for this space was found in your Strategy. " + ) if ignore_missing_space: logger.warning(msg + "This space will be ignored.") else: raise OperationalException( msg + f"Please make sure to have parameters for this space enabled for optimization " - f"or remove the '{space}' space from hyperoptimization.") + f"or remove the '{space}' space from hyperoptimization." + ) class HyperOptAuto(IHyperOpt): @@ -44,7 +47,7 @@ class HyperOptAuto(IHyperOpt): :param name: function name. :return: a requested function. """ - hyperopt_cls = getattr(self.strategy, 'HyperOpt', None) + hyperopt_cls = getattr(self.strategy, "HyperOpt", None) default_func = getattr(super(), name) if hyperopt_cls: return getattr(hyperopt_cls, name, default_func) @@ -63,36 +66,36 @@ class HyperOptAuto(IHyperOpt): return indicator_space else: _format_exception_message( - category, - self.config.get("hyperopt_ignore_missing_space", False)) + category, self.config.get("hyperopt_ignore_missing_space", False) + ) return [] - def buy_indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('buy') + def buy_indicator_space(self) -> List["Dimension"]: + return self._get_indicator_space("buy") - def sell_indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('sell') + def sell_indicator_space(self) -> List["Dimension"]: + return self._get_indicator_space("sell") - def protection_space(self) -> List['Dimension']: - return self._get_indicator_space('protection') + def protection_space(self) -> List["Dimension"]: + return self._get_indicator_space("protection") def generate_roi_table(self, params: Dict) -> Dict[int, float]: - return self._get_func('generate_roi_table')(params) + return self._get_func("generate_roi_table")(params) - def roi_space(self) -> List['Dimension']: - return self._get_func('roi_space')() + def roi_space(self) -> List["Dimension"]: + return self._get_func("roi_space")() - def stoploss_space(self) -> List['Dimension']: - return self._get_func('stoploss_space')() + def stoploss_space(self) -> List["Dimension"]: + return self._get_func("stoploss_space")() def generate_trailing_params(self, params: Dict) -> Dict: - return self._get_func('generate_trailing_params')(params) + return self._get_func("generate_trailing_params")(params) - def trailing_space(self) -> List['Dimension']: - return self._get_func('trailing_space')() + def trailing_space(self) -> List["Dimension"]: + return self._get_func("trailing_space")() - def max_open_trades_space(self) -> List['Dimension']: - return self._get_func('max_open_trades_space')() + def max_open_trades_space(self) -> List["Dimension"]: + return self._get_func("max_open_trades_space")() - def generate_estimator(self, dimensions: List['Dimension'], **kwargs) -> EstimatorType: - return self._get_func('generate_estimator')(dimensions=dimensions, **kwargs) + def generate_estimator(self, dimensions: List["Dimension"], **kwargs) -> EstimatorType: + return self._get_func("generate_estimator")(dimensions=dimensions, **kwargs) diff --git a/freqtrade/optimize/hyperopt_epoch_filters.py b/freqtrade/optimize/hyperopt_epoch_filters.py index 80cc89d4b..0a88b9d65 100644 --- a/freqtrade/optimize/hyperopt_epoch_filters.py +++ b/freqtrade/optimize/hyperopt_epoch_filters.py @@ -11,11 +11,10 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) """ Filter our items from the list of hyperopt results """ - if filteroptions['only_best']: - epochs = [x for x in epochs if x['is_best']] - if filteroptions['only_profitable']: - epochs = [x for x in epochs - if x['results_metrics'].get('profit_total', 0) > 0] + if filteroptions["only_best"]: + epochs = [x for x in epochs if x["is_best"]] + if filteroptions["only_profitable"]: + epochs = [x for x in epochs if x["results_metrics"].get("profit_total", 0) > 0] epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) @@ -25,10 +24,12 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions) if log: - logger.info(f"{len(epochs)} " + - ("best " if filteroptions['only_best'] else "") + - ("profitable " if filteroptions['only_profitable'] else "") + - "epochs found.") + logger.info( + f"{len(epochs)} " + + ("best " if filteroptions["only_best"] else "") + + ("profitable " if filteroptions["only_profitable"] else "") + + "epochs found." + ) return epochs @@ -36,93 +37,87 @@ def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int): """ Filter epochs with trade-counts > trades """ - return [ - x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count - ] + return [x for x in epochs if x["results_metrics"].get("total_trades", 0) > trade_count] def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List: + if filteroptions["filter_min_trades"] > 0: + epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions["filter_min_trades"]) - if filteroptions['filter_min_trades'] > 0: - epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades']) - - if filteroptions['filter_max_trades'] > 0: + if filteroptions["filter_max_trades"] > 0: epochs = [ - x for x in epochs - if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades'] + x + for x in epochs + if x["results_metrics"].get("total_trades") < filteroptions["filter_max_trades"] ] return epochs def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: - def get_duration_value(x): # Duration in minutes ... - if 'holding_avg_s' in x['results_metrics']: - avg = x['results_metrics']['holding_avg_s'] + if "holding_avg_s" in x["results_metrics"]: + avg = x["results_metrics"]["holding_avg_s"] return avg // 60 raise OperationalException( "Holding-average not available. Please omit the filter on average time, " - "or rerun hyperopt with this version") + "or rerun hyperopt with this version" + ) - if filteroptions['filter_min_avg_time'] is not None: + if filteroptions["filter_min_avg_time"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if get_duration_value(x) > filteroptions['filter_min_avg_time'] - ] - if filteroptions['filter_max_avg_time'] is not None: + epochs = [x for x in epochs if get_duration_value(x) > filteroptions["filter_min_avg_time"]] + if filteroptions["filter_max_avg_time"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [ - x for x in epochs - if get_duration_value(x) < filteroptions['filter_max_avg_time'] - ] + epochs = [x for x in epochs if get_duration_value(x) < filteroptions["filter_max_avg_time"]] return epochs def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: - - if filteroptions['filter_min_avg_profit'] is not None: + if filteroptions["filter_min_avg_profit"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ - x for x in epochs - if x['results_metrics'].get('profit_mean', 0) * 100 - > filteroptions['filter_min_avg_profit'] + x + for x in epochs + if x["results_metrics"].get("profit_mean", 0) * 100 + > filteroptions["filter_min_avg_profit"] ] - if filteroptions['filter_max_avg_profit'] is not None: + if filteroptions["filter_max_avg_profit"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ - x for x in epochs - if x['results_metrics'].get('profit_mean', 0) * 100 - < filteroptions['filter_max_avg_profit'] + x + for x in epochs + if x["results_metrics"].get("profit_mean", 0) * 100 + < filteroptions["filter_max_avg_profit"] ] - if filteroptions['filter_min_total_profit'] is not None: + if filteroptions["filter_min_total_profit"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ - x for x in epochs - if x['results_metrics'].get('profit_total_abs', 0) - > filteroptions['filter_min_total_profit'] + x + for x in epochs + if x["results_metrics"].get("profit_total_abs", 0) + > filteroptions["filter_min_total_profit"] ] - if filteroptions['filter_max_total_profit'] is not None: + if filteroptions["filter_max_total_profit"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ - x for x in epochs - if x['results_metrics'].get('profit_total_abs', 0) - < filteroptions['filter_max_total_profit'] + x + for x in epochs + if x["results_metrics"].get("profit_total_abs", 0) + < filteroptions["filter_max_total_profit"] ] return epochs def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List: - - if filteroptions['filter_min_objective'] is not None: + if filteroptions["filter_min_objective"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']] - if filteroptions['filter_max_objective'] is not None: + epochs = [x for x in epochs if x["loss"] < filteroptions["filter_min_objective"]] + if filteroptions["filter_max_objective"] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) - epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']] + epochs = [x for x in epochs if x["loss"] > filteroptions["filter_max_objective"]] return epochs diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 65dd7ed87..216e40753 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -2,6 +2,7 @@ IHyperOpt interface This module defines the interface to apply for hyperopt """ + import logging import math from abc import ABC @@ -30,6 +31,7 @@ class IHyperOpt(ABC): Class attributes you can use: timeframe -> int: value of the timeframe to use for the strategy """ + timeframe: str strategy: IStrategy @@ -37,7 +39,7 @@ class IHyperOpt(ABC): self.config = config # Assign timeframe to be used in hyperopt - IHyperOpt.timeframe = str(config['timeframe']) + IHyperOpt.timeframe = str(config["timeframe"]) def generate_estimator(self, dimensions: List[Dimension], **kwargs) -> EstimatorType: """ @@ -45,7 +47,7 @@ class IHyperOpt(ABC): Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class inheriting from RegressorMixin (from sklearn). """ - return 'ET' + return "ET" def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ @@ -55,10 +57,10 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + roi_table[0] = params["roi_p1"] + params["roi_p2"] + params["roi_p3"] + roi_table[params["roi_t3"]] = params["roi_p1"] + params["roi_p2"] + roi_table[params["roi_t3"] + params["roi_t2"]] = params["roi_p1"] + roi_table[params["roi_t3"] + params["roi_t2"] + params["roi_t1"]] = 0 return roi_table @@ -96,49 +98,52 @@ class IHyperOpt(ABC): roi_t_scale = timeframe_min / 5 roi_p_scale = math.log1p(timeframe_min) / math.log1p(5) roi_limits = { - 'roi_t1_min': int(10 * roi_t_scale * roi_t_alpha), - 'roi_t1_max': int(120 * roi_t_scale * roi_t_alpha), - 'roi_t2_min': int(10 * roi_t_scale * roi_t_alpha), - 'roi_t2_max': int(60 * roi_t_scale * roi_t_alpha), - 'roi_t3_min': int(10 * roi_t_scale * roi_t_alpha), - 'roi_t3_max': int(40 * roi_t_scale * roi_t_alpha), - 'roi_p1_min': 0.01 * roi_p_scale * roi_p_alpha, - 'roi_p1_max': 0.04 * roi_p_scale * roi_p_alpha, - 'roi_p2_min': 0.01 * roi_p_scale * roi_p_alpha, - 'roi_p2_max': 0.07 * roi_p_scale * roi_p_alpha, - 'roi_p3_min': 0.01 * roi_p_scale * roi_p_alpha, - 'roi_p3_max': 0.20 * roi_p_scale * roi_p_alpha, + "roi_t1_min": int(10 * roi_t_scale * roi_t_alpha), + "roi_t1_max": int(120 * roi_t_scale * roi_t_alpha), + "roi_t2_min": int(10 * roi_t_scale * roi_t_alpha), + "roi_t2_max": int(60 * roi_t_scale * roi_t_alpha), + "roi_t3_min": int(10 * roi_t_scale * roi_t_alpha), + "roi_t3_max": int(40 * roi_t_scale * roi_t_alpha), + "roi_p1_min": 0.01 * roi_p_scale * roi_p_alpha, + "roi_p1_max": 0.04 * roi_p_scale * roi_p_alpha, + "roi_p2_min": 0.01 * roi_p_scale * roi_p_alpha, + "roi_p2_max": 0.07 * roi_p_scale * roi_p_alpha, + "roi_p3_min": 0.01 * roi_p_scale * roi_p_alpha, + "roi_p3_max": 0.20 * roi_p_scale * roi_p_alpha, } logger.debug(f"Using roi space limits: {roi_limits}") p = { - 'roi_t1': roi_limits['roi_t1_min'], - 'roi_t2': roi_limits['roi_t2_min'], - 'roi_t3': roi_limits['roi_t3_min'], - 'roi_p1': roi_limits['roi_p1_min'], - 'roi_p2': roi_limits['roi_p2_min'], - 'roi_p3': roi_limits['roi_p3_min'], + "roi_t1": roi_limits["roi_t1_min"], + "roi_t2": roi_limits["roi_t2_min"], + "roi_t3": roi_limits["roi_t3_min"], + "roi_p1": roi_limits["roi_p1_min"], + "roi_p2": roi_limits["roi_p2_min"], + "roi_p3": roi_limits["roi_p3_min"], } logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}") p = { - 'roi_t1': roi_limits['roi_t1_max'], - 'roi_t2': roi_limits['roi_t2_max'], - 'roi_t3': roi_limits['roi_t3_max'], - 'roi_p1': roi_limits['roi_p1_max'], - 'roi_p2': roi_limits['roi_p2_max'], - 'roi_p3': roi_limits['roi_p3_max'], + "roi_t1": roi_limits["roi_t1_max"], + "roi_t2": roi_limits["roi_t2_max"], + "roi_t3": roi_limits["roi_t3_max"], + "roi_p1": roi_limits["roi_p1_max"], + "roi_p2": roi_limits["roi_p2_max"], + "roi_p3": roi_limits["roi_p3_max"], } logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}") return [ - Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'), - Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'), - Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'), - SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3, - name='roi_p1'), - SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3, - name='roi_p2'), - SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3, - name='roi_p3'), + Integer(roi_limits["roi_t1_min"], roi_limits["roi_t1_max"], name="roi_t1"), + Integer(roi_limits["roi_t2_min"], roi_limits["roi_t2_max"], name="roi_t2"), + Integer(roi_limits["roi_t3_min"], roi_limits["roi_t3_max"], name="roi_t3"), + SKDecimal( + roi_limits["roi_p1_min"], roi_limits["roi_p1_max"], decimals=3, name="roi_p1" + ), + SKDecimal( + roi_limits["roi_p2_min"], roi_limits["roi_p2_max"], decimals=3, name="roi_p2" + ), + SKDecimal( + roi_limits["roi_p3_min"], roi_limits["roi_p3_max"], decimals=3, name="roi_p3" + ), ] def stoploss_space(self) -> List[Dimension]: @@ -149,7 +154,7 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ return [ - SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), + SKDecimal(-0.35, -0.02, decimals=3, name="stoploss"), ] def generate_trailing_params(self, params: Dict) -> Dict: @@ -157,11 +162,12 @@ class IHyperOpt(ABC): Create dict with trailing stop parameters. """ return { - 'trailing_stop': params['trailing_stop'], - 'trailing_stop_positive': params['trailing_stop_positive'], - 'trailing_stop_positive_offset': (params['trailing_stop_positive'] + - params['trailing_stop_positive_offset_p1']), - 'trailing_only_offset_is_reached': params['trailing_only_offset_is_reached'], + "trailing_stop": params["trailing_stop"], + "trailing_stop_positive": params["trailing_stop_positive"], + "trailing_stop_positive_offset": ( + params["trailing_stop_positive"] + params["trailing_stop_positive_offset_p1"] + ), + "trailing_only_offset_is_reached": params["trailing_only_offset_is_reached"], } def trailing_space(self) -> List[Dimension]: @@ -177,18 +183,15 @@ class IHyperOpt(ABC): # This parameter is included into the hyperspace dimensions rather than assigning # it explicitly in the code in order to have it printed in the results along with # other 'trailing' hyperspace parameters. - Categorical([True], name='trailing_stop'), - - SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), - + Categorical([True], name="trailing_stop"), + SKDecimal(0.01, 0.35, decimals=3, name="trailing_stop_positive"), # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', # so this intermediate parameter is used as the value of the difference between # them. The value of the 'trailing_stop_positive_offset' is constructed in the # generate_trailing_params() method. # This is similar to the hyperspace dimensions used for constructing the ROI tables. - SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), - - Categorical([True, False], name='trailing_only_offset_is_reached'), + SKDecimal(0.001, 0.1, decimals=3, name="trailing_stop_positive_offset_p1"), + Categorical([True, False], name="trailing_only_offset_is_reached"), ] def max_open_trades_space(self) -> List[Dimension]: @@ -198,7 +201,7 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ return [ - Integer(-1, 10, name='max_open_trades'), + Integer(-1, 10, name="max_open_trades"), ] # This is needed for proper unpickling the class attribute timeframe @@ -206,9 +209,9 @@ class IHyperOpt(ABC): # Why do I still need such shamanic mantras in modern python? def __getstate__(self): state = self.__dict__.copy() - state['timeframe'] = self.timeframe + state["timeframe"] = self.timeframe return state def __setstate__(self, state): self.__dict__.update(state) - IHyperOpt.timeframe = state['timeframe'] + IHyperOpt.timeframe = state["timeframe"] diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index d7b30dfd3..39457b753 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -17,15 +17,22 @@ class IHyperOptLoss(ABC): Interface for freqtrade hyperopt Loss functions. Defines the custom loss function (`hyperopt_loss_function()` which is evaluated every epoch.) """ + timeframe: str @staticmethod @abstractmethod - def hyperopt_loss_function(*, results: DataFrame, trade_count: int, - min_date: datetime, max_date: datetime, - config: Config, processed: Dict[str, DataFrame], - backtest_stats: Dict[str, Any], - **kwargs) -> float: + def hyperopt_loss_function( + *, + results: DataFrame, + trade_count: int, + min_date: datetime, + max_date: datetime, + config: Config, + processed: Dict[str, DataFrame], + backtest_stats: Dict[str, Any], + **kwargs, + ) -> float: """ Objective function, returns smaller number for better results """ diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 5a09d92b5..a7c03b93f 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -37,7 +37,8 @@ def hyperopt_serializer(x): class HyperoptStateContainer: - """ Singleton class to track state of hyperopt""" + """Singleton class to track state of hyperopt""" + state: HyperoptState = HyperoptState.OPTIMIZE @classmethod @@ -46,20 +47,21 @@ class HyperoptStateContainer: class HyperoptTools: - @staticmethod def get_strategy_filename(config: Config, strategy_name: str) -> Optional[Path]: """ Get Strategy-location (filename) from strategy_name """ from freqtrade.resolvers.strategy_resolver import StrategyResolver + strategy_objs = StrategyResolver.search_all_objects( - config, False, config.get('recursive_strategy_search', False)) - strategies = [s for s in strategy_objs if s['name'] == strategy_name] + config, False, config.get("recursive_strategy_search", False) + ) + strategies = [s for s in strategy_objs if s["name"] == strategy_name] if strategies: strategy = strategies[0] - return Path(strategy['location']) + return Path(strategy["location"]) return None @staticmethod @@ -67,37 +69,40 @@ class HyperoptTools: """ Generate files """ - final_params = deepcopy(params['params_not_optimized']) - final_params = deep_merge_dicts(params['params_details'], final_params) + final_params = deepcopy(params["params_not_optimized"]) + final_params = deep_merge_dicts(params["params_details"], final_params) final_params = { - 'strategy_name': strategy_name, - 'params': final_params, - 'ft_stratparam_v': 1, - 'export_time': datetime.now(timezone.utc), + "strategy_name": strategy_name, + "params": final_params, + "ft_stratparam_v": 1, + "export_time": datetime.now(timezone.utc), } logger.info(f"Dumping parameters to {filename}") - with filename.open('w') as f: - rapidjson.dump(final_params, f, indent=2, - default=hyperopt_serializer, - number_mode=HYPER_PARAMS_FILE_FORMAT - ) + with filename.open("w") as f: + rapidjson.dump( + final_params, + f, + indent=2, + default=hyperopt_serializer, + number_mode=HYPER_PARAMS_FILE_FORMAT, + ) @staticmethod def load_params(filename: Path) -> Dict: """ Load parameters from file """ - with filename.open('r') as f: + with filename.open("r") as f: params = rapidjson.load(f, number_mode=HYPER_PARAMS_FILE_FORMAT) return params @staticmethod def try_export_params(config: Config, strategy_name: str, params: Dict): - if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): + if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get("disableparamexport", False): # Export parameters ... fn = HyperoptTools.get_strategy_filename(config, strategy_name) if fn: - HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json')) + HyperoptTools.export_params(params, strategy_name, fn.with_suffix(".json")) else: logger.warning("Strategy not found, not exporting parameter file.") @@ -107,10 +112,10 @@ class HyperoptTools: Tell if the space value is contained in the configuration """ # 'trailing' and 'protection spaces are not included in the 'default' set of spaces - if space in ('trailing', 'protection', 'trades'): - return any(s in config['spaces'] for s in [space, 'all']) + if space in ("trailing", "protection", "trades"): + return any(s in config["spaces"] for s in [space, "all"]) else: - return any(s in config['spaces'] for s in [space, 'all', 'default']) + return any(s in config["spaces"] for s in [space, "all", "default"]) @staticmethod def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]: @@ -118,8 +123,9 @@ class HyperoptTools: Stream hyperopt results from file """ import rapidjson + logger.info(f"Reading epochs from '{results_file}'") - with results_file.open('r') as f: + with results_file.open("r") as f: data = [] for line in f: data += [rapidjson.loads(line)] @@ -131,7 +137,7 @@ class HyperoptTools: @staticmethod def _test_hyperopt_results_exist(results_file) -> bool: if results_file.is_file() and results_file.stat().st_size > 0: - if results_file.suffix == '.pickle': + if results_file.suffix == ".pickle": raise OperationalException( "Legacy hyperopt results are no longer supported." "Please rerun hyperopt or use an older version to load this file." @@ -144,18 +150,18 @@ class HyperoptTools: @staticmethod def load_filtered_results(results_file: Path, config: Config) -> Tuple[List, int]: filteroptions = { - 'only_best': config.get('hyperopt_list_best', False), - 'only_profitable': config.get('hyperopt_list_profitable', False), - 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), - 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), - 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time'), - 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time'), - 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit'), - 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit'), - 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit'), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit'), - 'filter_min_objective': config.get('hyperopt_list_min_objective'), - 'filter_max_objective': config.get('hyperopt_list_max_objective'), + "only_best": config.get("hyperopt_list_best", False), + "only_profitable": config.get("hyperopt_list_profitable", False), + "filter_min_trades": config.get("hyperopt_list_min_trades", 0), + "filter_max_trades": config.get("hyperopt_list_max_trades", 0), + "filter_min_avg_time": config.get("hyperopt_list_min_avg_time"), + "filter_max_avg_time": config.get("hyperopt_list_max_avg_time"), + "filter_min_avg_profit": config.get("hyperopt_list_min_avg_profit"), + "filter_max_avg_profit": config.get("hyperopt_list_max_avg_profit"), + "filter_min_total_profit": config.get("hyperopt_list_min_total_profit"), + "filter_max_total_profit": config.get("hyperopt_list_max_total_profit"), + "filter_min_objective": config.get("hyperopt_list_min_objective"), + "filter_max_objective": config.get("hyperopt_list_max_objective"), } if not HyperoptTools._test_hyperopt_results_exist(results_file): # No file found. @@ -165,10 +171,11 @@ class HyperoptTools: epochs = [] total_epochs = 0 for epochs_tmp in HyperoptTools._read_results(results_file): - if total_epochs == 0 and epochs_tmp[0].get('is_best') is None: + if total_epochs == 0 and epochs_tmp[0].get("is_best") is None: raise OperationalException( "The file with HyperoptTools results is incompatible with this version " - "of Freqtrade and cannot be loaded.") + "of Freqtrade and cannot be loaded." + ) total_epochs += len(epochs_tmp) epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False) @@ -180,13 +187,18 @@ class HyperoptTools: return epochs, total_epochs @staticmethod - def show_epoch_details(results, total_epochs: int, print_json: bool, - no_header: bool = False, header_str: Optional[str] = None) -> None: + def show_epoch_details( + results, + total_epochs: int, + print_json: bool, + no_header: bool = False, + header_str: Optional[str] = None, + ) -> None: """ Display details of the hyperopt result """ - params = results.get('params_details', {}) - non_optimized = results.get('params_not_optimized', {}) + params = results.get("params_details", {}) + non_optimized = results.get("params_not_optimized", {}) # Default header string if header_str is None: @@ -198,23 +210,34 @@ class HyperoptTools: if print_json: result_dict: Dict = {} - for s in ['buy', 'sell', 'protection', - 'roi', 'stoploss', 'trailing', 'max_open_trades']: + for s in [ + "buy", + "sell", + "protection", + "roi", + "stoploss", + "trailing", + "max_open_trades", + ]: HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) print(rapidjson.dumps(result_dict, default=str, number_mode=HYPER_PARAMS_FILE_FORMAT)) else: - HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:", - non_optimized) - HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:", - non_optimized) - HyperoptTools._params_pretty_print(params, 'protection', - "Protection hyperspace params:", non_optimized) - HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized) - HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized) - HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized) HyperoptTools._params_pretty_print( - params, 'max_open_trades', "Max Open Trades:", non_optimized) + params, "buy", "Buy hyperspace params:", non_optimized + ) + HyperoptTools._params_pretty_print( + params, "sell", "Sell hyperspace params:", non_optimized + ) + HyperoptTools._params_pretty_print( + params, "protection", "Protection hyperspace params:", non_optimized + ) + HyperoptTools._params_pretty_print(params, "roi", "ROI table:", non_optimized) + HyperoptTools._params_pretty_print(params, "stoploss", "Stoploss:", non_optimized) + HyperoptTools._params_pretty_print(params, "trailing", "Trailing stop:", non_optimized) + HyperoptTools._params_pretty_print( + params, "max_open_trades", "Max Open Trades:", non_optimized + ) @staticmethod def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: @@ -227,23 +250,23 @@ class HyperoptTools: if len(space_non_optimized) > 0: all_space_params = {**space_params, **space_non_optimized} - if space in ['buy', 'sell']: - result_dict.setdefault('params', {}).update(all_space_params) - elif space == 'roi': + if space in ["buy", "sell"]: + result_dict.setdefault("params", {}).update(all_space_params) + elif space == "roi": # Convert keys in min_roi dict to strings because # rapidjson cannot dump dicts with integer keys... - result_dict['minimal_roi'] = {str(k): v for k, v in all_space_params.items()} + result_dict["minimal_roi"] = {str(k): v for k, v in all_space_params.items()} else: # 'stoploss', 'trailing' result_dict.update(all_space_params) @staticmethod def _params_pretty_print( - params, space: str, header: str, non_optimized: Optional[Dict] = None) -> None: - + params, space: str, header: str, non_optimized: Optional[Dict] = None + ) -> None: if space in params or (non_optimized and space in non_optimized): space_params = HyperoptTools._space_params(params, space, 5) no_params = HyperoptTools._space_params(non_optimized, space, 5) - appendix = '' + appendix = "" if not space_params and not no_params: # No parameters - don't print return @@ -254,15 +277,18 @@ class HyperoptTools: result = f"\n# {header}\n" if space == "stoploss": stoploss = safe_value_fallback2(space_params, no_params, space, space) - result += (f"stoploss = {stoploss}{appendix}") + result += f"stoploss = {stoploss}{appendix}" elif space == "max_open_trades": max_open_trades = safe_value_fallback2(space_params, no_params, space, space) - result += (f"max_open_trades = {max_open_trades}{appendix}") + result += f"max_open_trades = {max_open_trades}{appendix}" elif space == "roi": - result = result[:-1] + f'{appendix}\n' - minimal_roi_result = rapidjson.dumps({ - str(k): v for k, v in (space_params or no_params).items() - }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) + result = result[:-1] + f"{appendix}\n" + minimal_roi_result = rapidjson.dumps( + {str(k): v for k, v in (space_params or no_params).items()}, + default=str, + indent=4, + number_mode=rapidjson.NM_NATIVE, + ) result += f"minimal_roi = {minimal_roi_result}" elif space == "trailing": for k, v in (space_params or no_params).items(): @@ -291,177 +317,213 @@ class HyperoptTools: """ p = params.copy() p.update(non_optimized) - result = '{\n' + result = "{\n" for k, param in p.items(): result += " " * indent + f'"{k}": ' - result += f'"{param}",' if isinstance(param, str) else f'{param},' + result += f'"{param}",' if isinstance(param, str) else f"{param}," if k in non_optimized: result += NON_OPT_PARAM_APPENDIX result += "\n" - result += '}' + result += "}" return result @staticmethod def is_best_loss(results, current_best_loss: float) -> bool: - return bool(results['loss'] < current_best_loss) + return bool(results["loss"] < current_best_loss) @staticmethod def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str: """ Return the formatted results explanation in a string """ - return (f"{results_metrics['total_trades']:6d} trades. " - f"{results_metrics['wins']}/{results_metrics['draws']}" - f"/{results_metrics['losses']} Wins/Draws/Losses. " - f"Avg profit {results_metrics['profit_mean']:7.2%}. " - f"Median profit {results_metrics['profit_median']:7.2%}. " - f"Total profit {results_metrics['profit_total_abs']:11.8f} {stake_currency} " - f"({results_metrics['profit_total']:8.2%}). " - f"Avg duration {results_metrics['holding_avg']} min." - ) + return ( + f"{results_metrics['total_trades']:6d} trades. " + f"{results_metrics['wins']}/{results_metrics['draws']}" + f"/{results_metrics['losses']} Wins/Draws/Losses. " + f"Avg profit {results_metrics['profit_mean']:7.2%}. " + f"Median profit {results_metrics['profit_median']:7.2%}. " + f"Total profit {results_metrics['profit_total_abs']:11.8f} {stake_currency} " + f"({results_metrics['profit_total']:8.2%}). " + f"Avg duration {results_metrics['holding_avg']} min." + ) @staticmethod def _format_explanation_string(results, total_epochs) -> str: - return (("*" if results['is_initial_point'] else " ") + - f"{results['current_epoch']:5d}/{total_epochs}: " + - f"{results['results_explanation']} " + - f"Objective: {results['loss']:.5f}") + return ( + ("*" if results["is_initial_point"] else " ") + + f"{results['current_epoch']:5d}/{total_epochs}: " + + f"{results['results_explanation']} " + + f"Objective: {results['loss']:.5f}" + ) @staticmethod def prepare_trials_columns(trials: pd.DataFrame, has_drawdown: bool) -> pd.DataFrame: - trials['Best'] = '' + trials["Best"] = "" - if 'results_metrics.winsdrawslosses' not in trials.columns: + if "results_metrics.winsdrawslosses" not in trials.columns: # Ensure compatibility with older versions of hyperopt results - trials['results_metrics.winsdrawslosses'] = 'N/A' + trials["results_metrics.winsdrawslosses"] = "N/A" if not has_drawdown: # Ensure compatibility with older versions of hyperopt results - trials['results_metrics.max_drawdown_account'] = None - if 'is_random' not in trials.columns: - trials['is_random'] = False + trials["results_metrics.max_drawdown_account"] = None + if "is_random" not in trials.columns: + trials["is_random"] = False # New mode, using backtest result for metrics - trials['results_metrics.winsdrawslosses'] = trials.apply( + trials["results_metrics.winsdrawslosses"] = trials.apply( lambda x: generate_wins_draws_losses( - x['results_metrics.wins'], x['results_metrics.draws'], - x['results_metrics.losses'] - ), axis=1) + x["results_metrics.wins"], x["results_metrics.draws"], x["results_metrics.losses"] + ), + axis=1, + ) - trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades', - 'results_metrics.winsdrawslosses', - 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', - 'results_metrics.profit_total', 'results_metrics.holding_avg', - 'results_metrics.max_drawdown', - 'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs', - 'loss', 'is_initial_point', 'is_random', 'is_best']] + trials = trials[ + [ + "Best", + "current_epoch", + "results_metrics.total_trades", + "results_metrics.winsdrawslosses", + "results_metrics.profit_mean", + "results_metrics.profit_total_abs", + "results_metrics.profit_total", + "results_metrics.holding_avg", + "results_metrics.max_drawdown", + "results_metrics.max_drawdown_account", + "results_metrics.max_drawdown_abs", + "loss", + "is_initial_point", + "is_random", + "is_best", + ] + ] trials.columns = [ - 'Best', 'Epoch', 'Trades', ' Win Draw Loss Win%', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account', - 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_random', 'is_best' - ] + "Best", + "Epoch", + "Trades", + " Win Draw Loss Win%", + "Avg profit", + "Total profit", + "Profit", + "Avg duration", + "max_drawdown", + "max_drawdown_account", + "max_drawdown_abs", + "Objective", + "is_initial_point", + "is_random", + "is_best", + ] return trials @staticmethod - def get_result_table(config: Config, results: list, total_epochs: int, highlight_best: bool, - print_colorized: bool, remove_header: int) -> str: + def get_result_table( + config: Config, + results: list, + total_epochs: int, + highlight_best: bool, + print_colorized: bool, + remove_header: int, + ) -> str: """ Log result table """ if not results: - return '' + return "" tabulate.PRESERVE_WHITESPACE = True trials = json_normalize(results, max_level=1) - has_account_drawdown = 'results_metrics.max_drawdown_account' in trials.columns + has_account_drawdown = "results_metrics.max_drawdown_account" in trials.columns trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown) - trials['is_profit'] = False - trials.loc[trials['is_initial_point'] | trials['is_random'], 'Best'] = '* ' - trials.loc[trials['is_best'], 'Best'] = 'Best' + trials["is_profit"] = False + trials.loc[trials["is_initial_point"] | trials["is_random"], "Best"] = "* " + trials.loc[trials["is_best"], "Best"] = "Best" trials.loc[ - (trials['is_initial_point'] | trials['is_random']) & trials['is_best'], - 'Best'] = '* Best' - trials.loc[trials['Total profit'] > 0, 'is_profit'] = True - trials['Trades'] = trials['Trades'].astype(str) + (trials["is_initial_point"] | trials["is_random"]) & trials["is_best"], "Best" + ] = "* Best" + trials.loc[trials["Total profit"] > 0, "is_profit"] = True + trials["Trades"] = trials["Trades"].astype(str) # perc_multi = 1 if legacy_mode else 100 - trials['Epoch'] = trials['Epoch'].apply( - lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs) + trials["Epoch"] = trials["Epoch"].apply( + lambda x: "{}/{}".format(str(x).rjust(len(str(total_epochs)), " "), total_epochs) ) - trials['Avg profit'] = trials['Avg profit'].apply( - lambda x: f'{x:,.2%}'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + trials["Avg profit"] = trials["Avg profit"].apply( + lambda x: f"{x:,.2%}".rjust(7, " ") if not isna(x) else "--".rjust(7, " ") ) - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}" - if not isna(x) else "--".rjust(7, ' ') + trials["Avg duration"] = trials["Avg duration"].apply( + lambda x: f"{x:,.1f} m".rjust(7, " ") + if isinstance(x, float) + else f"{x}" + if not isna(x) + else "--".rjust(7, " ") ) - trials['Objective'] = trials['Objective'].apply( - lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') + trials["Objective"] = trials["Objective"].apply( + lambda x: f"{x:,.5f}".rjust(8, " ") if x != 100000 else "N/A".rjust(8, " ") ) - stake_currency = config['stake_currency'] + stake_currency = config["stake_currency"] trials[f"Max Drawdown{' (Acct)' if has_account_drawdown else ''}"] = trials.apply( lambda x: "{} {}".format( - fmt_coin(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True), - (f"({x['max_drawdown_account']:,.2%})" + fmt_coin(x["max_drawdown_abs"], stake_currency, keep_trailing_zeros=True), + ( + f"({x['max_drawdown_account']:,.2%})" if has_account_drawdown else f"({x['max_drawdown']:,.2%})" - ).rjust(10, ' ') + ).rjust(10, " "), ).rjust(25 + len(stake_currency)) - if x['max_drawdown'] != 0.0 or x['max_drawdown_account'] != 0.0 - else '--'.rjust(25 + len(stake_currency)), - axis=1 + if x["max_drawdown"] != 0.0 or x["max_drawdown_account"] != 0.0 + else "--".rjust(25 + len(stake_currency)), + axis=1, ) - trials = trials.drop(columns=['max_drawdown_abs', 'max_drawdown', 'max_drawdown_account']) + trials = trials.drop(columns=["max_drawdown_abs", "max_drawdown", "max_drawdown_account"]) - trials['Profit'] = trials.apply( - lambda x: '{} {}'.format( - fmt_coin(x['Total profit'], stake_currency, keep_trailing_zeros=True), - f"({x['Profit']:,.2%})".rjust(10, ' ') + trials["Profit"] = trials.apply( + lambda x: "{} {}".format( + fmt_coin(x["Total profit"], stake_currency, keep_trailing_zeros=True), + f"({x['Profit']:,.2%})".rjust(10, " "), ).rjust(25 + len(stake_currency)) - if x['Total profit'] != 0.0 else '--'.rjust(25 + len(stake_currency)), - axis=1 + if x["Total profit"] != 0.0 + else "--".rjust(25 + len(stake_currency)), + axis=1, ) - trials = trials.drop(columns=['Total profit']) + trials = trials.drop(columns=["Total profit"]) if print_colorized: trials2 = trials.astype(str) for i in range(len(trials)): - if trials.loc[i]['is_profit']: + if trials.loc[i]["is_profit"]: for j in range(len(trials.loc[i]) - 3): trials2.iat[i, j] = f"{Fore.GREEN}{str(trials.iloc[i, j])}{Fore.RESET}" - if trials.loc[i]['is_best'] and highlight_best: + if trials.loc[i]["is_best"] and highlight_best: for j in range(len(trials.loc[i]) - 3): trials2.iat[i, j] = ( f"{Style.BRIGHT}{str(trials.iloc[i, j])}{Style.RESET_ALL}" ) trials = trials2 del trials2 - trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit', 'is_random']) + trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit", "is_random"]) if remove_header > 0: table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='orgtbl', - headers='keys', stralign="right" + trials.to_dict(orient="list"), tablefmt="orgtbl", headers="keys", stralign="right" ) table = table.split("\n", remove_header)[remove_header] elif remove_header < 0: table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='psql', - headers='keys', stralign="right" + trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" ) table = "\n".join(table.split("\n")[0:remove_header]) else: table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='psql', - headers='keys', stralign="right" + trials.to_dict(orient="list"), tablefmt="psql", headers="keys", stralign="right" ) return table @@ -479,56 +541,75 @@ class HyperoptTools: return try: - Path(csv_file).open('w+').close() + Path(csv_file).open("w+").close() except OSError: logger.error(f"Failed to create CSV file: {csv_file}") return trials = json_normalize(results, max_level=1) - trials['Best'] = '' - trials['Stake currency'] = config['stake_currency'] + trials["Best"] = "" + trials["Stake currency"] = config["stake_currency"] - base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades', - 'results_metrics.profit_mean', 'results_metrics.profit_median', - 'results_metrics.profit_total', 'Stake currency', - 'results_metrics.profit_total_abs', 'results_metrics.holding_avg', - 'results_metrics.trade_count_long', 'results_metrics.trade_count_short', - 'loss', 'is_initial_point', 'is_best'] + base_metrics = [ + "Best", + "current_epoch", + "results_metrics.total_trades", + "results_metrics.profit_mean", + "results_metrics.profit_median", + "results_metrics.profit_total", + "Stake currency", + "results_metrics.profit_total_abs", + "results_metrics.holding_avg", + "results_metrics.trade_count_long", + "results_metrics.trade_count_short", + "loss", + "is_initial_point", + "is_best", + ] perc_multi = 100 - param_metrics = [("params_dict." + param) for param in results[0]['params_dict'].keys()] + param_metrics = [("params_dict." + param) for param in results[0]["params_dict"].keys()] trials = trials[base_metrics + param_metrics] - base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit', - 'Stake currency', 'Profit', 'Avg duration', - 'Trade count long', 'Trade count short', - 'Objective', - 'is_initial_point', 'is_best'] - param_columns = list(results[0]['params_dict'].keys()) + base_columns = [ + "Best", + "Epoch", + "Trades", + "Avg profit", + "Median profit", + "Total profit", + "Stake currency", + "Profit", + "Avg duration", + "Trade count long", + "Trade count short", + "Objective", + "is_initial_point", + "is_best", + ] + param_columns = list(results[0]["params_dict"].keys()) trials.columns = base_columns + param_columns - trials['is_profit'] = False - trials.loc[trials['is_initial_point'], 'Best'] = '*' - trials.loc[trials['is_best'], 'Best'] = 'Best' - trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' - trials.loc[trials['Total profit'] > 0, 'is_profit'] = True - trials['Epoch'] = trials['Epoch'].astype(str) - trials['Trades'] = trials['Trades'].astype(str) - trials['Median profit'] = trials['Median profit'] * perc_multi + trials["is_profit"] = False + trials.loc[trials["is_initial_point"], "Best"] = "*" + trials.loc[trials["is_best"], "Best"] = "Best" + trials.loc[trials["is_initial_point"] & trials["is_best"], "Best"] = "* Best" + trials.loc[trials["Total profit"] > 0, "is_profit"] = True + trials["Epoch"] = trials["Epoch"].astype(str) + trials["Trades"] = trials["Trades"].astype(str) + trials["Median profit"] = trials["Median profit"] * perc_multi - trials['Total profit'] = trials['Total profit'].apply( - lambda x: f'{x:,.8f}' if x != 0.0 else "" + trials["Total profit"] = trials["Total profit"].apply( + lambda x: f"{x:,.8f}" if x != 0.0 else "" ) - trials['Profit'] = trials['Profit'].apply( - lambda x: f'{x:,.2f}' if not isna(x) else "" + trials["Profit"] = trials["Profit"].apply(lambda x: f"{x:,.2f}" if not isna(x) else "") + trials["Avg profit"] = trials["Avg profit"].apply( + lambda x: f"{x * perc_multi:,.2f}%" if not isna(x) else "" ) - trials['Avg profit'] = trials['Avg profit'].apply( - lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else "" - ) - trials['Objective'] = trials['Objective'].apply( - lambda x: f'{x:,.5f}' if x != 100000 else "" + trials["Objective"] = trials["Objective"].apply( + lambda x: f"{x:,.5f}" if x != 100000 else "" ) - trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) - trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8') + trials = trials.drop(columns=["is_initial_point", "is_best", "is_profit"]) + trials.to_csv(csv_file, index=False, header=True, mode="w", encoding="UTF-8") logger.info(f"CSV file created: {csv_file}")