From c0811ae8969799857e987ed7e93d1c1e78dd3a2f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 15 Sep 2021 21:36:53 +0200 Subject: [PATCH 1/2] Add possibility to override estimator from within hyperopt --- docs/advanced-hyperopt.md | 32 ++++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 8 ++++-- freqtrade/optimize/hyperopt_auto.py | 5 +++- freqtrade/optimize/hyperopt_interface.py | 13 +++++++++- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index f2f52b7dd..f5a52ff49 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy): !!! Note All overrides are optional and can be mixed/matched as necessary. +### Overriding Base estimator + +You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass. + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + return "RF" + +``` + +Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`". + +Some research will be necessary to find additional Regressors. + +Example for `ExtraTreesRegressor` ("ET") with additional parameters: + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + def generate_estimator(): + from skopt.learning import ExtraTreesRegressor + # Corresponds to "ET" - but allows additional parameters. + return ExtraTreesRegressor(n_estimators=100) + +``` + +!!! Note + While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used. + If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters. + ## Space options For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d047b7311..56d11934a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -365,10 +365,14 @@ class Hyperopt: } def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: + estimator = self.custom_hyperopt.generate_estimator() + logger.info(f"Using estimator {estimator}.") + # TODO: Impact of changing acq_optimizer to "sampling" is unclear + # (other than that it fails with other optimizers when using custom sklearn regressors) return Optimizer( dimensions, - base_estimator="ET", - acq_optimizer="auto", + base_estimator=estimator, + acq_optimizer="sampling", n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 80fe090c2..c1c769c72 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException with suppress(ImportError): from skopt.space import Dimension -from freqtrade.optimize.hyperopt_interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt def _format_exception_message(space: str) -> str: @@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt): def trailing_space(self) -> List['Dimension']: return self._get_func('trailing_space')() + + def generate_estimator(self) -> EstimatorType: + return self._get_func('generate_estimator')() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 8fb40f557..53b4f087c 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt import logging import math from abc import ABC -from typing import Dict, List +from typing import Dict, List, Union +from sklearn.base import RegressorMixin from skopt.space import Categorical, Dimension, Integer from freqtrade.exchange import timeframe_to_minutes @@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) +EstimatorType = Union[RegressorMixin, str] + class IHyperOpt(ABC): """ @@ -37,6 +40,14 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) + def generate_estimator(self) -> EstimatorType: + """ + Return base_estimator. + Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class + inheriting from RegressorMixin (from sklearn). + """ + return 'ET' + def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ Create a ROI table. From 994c3c3a4c5f36c02d249f4c13466b284c4991af Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Sep 2021 07:13:25 +0200 Subject: [PATCH 2/2] Add some errorhandling for custom estimator --- freqtrade/optimize/hyperopt.py | 14 ++++++++++---- tests/optimize/test_hyperopt.py | 4 ++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 56d11934a..9549b4054 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) -INITIAL_POINTS = 30 +INITIAL_POINTS = 5 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models # in the skopt model queue, to optimize memory consumption @@ -366,13 +366,19 @@ class Hyperopt: def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: estimator = self.custom_hyperopt.generate_estimator() + + acq_optimizer = "sampling" + if isinstance(estimator, str): + if estimator not in ("GP", "RF", "ET", "GBRT"): + raise OperationalException(f"Estimator {estimator} not supported.") + else: + acq_optimizer = "auto" + logger.info(f"Using estimator {estimator}.") - # TODO: Impact of changing acq_optimizer to "sampling" is unclear - # (other than that it fails with other optimizers when using custom sklearn regressors) return Optimizer( dimensions, base_estimator=estimator, - acq_optimizer="sampling", + acq_optimizer=acq_optimizer, n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b34c3a916..e4ce29d44 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.sell_rsi.value != 74 + hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1' + with pytest.raises(OperationalException, match="Estimator ET1 not supported."): + hyperopt.get_optimizer([], 2) + def test_SKDecimal(): space = SKDecimal(1, 2, decimals=2)