From 68ba1e1f3741d248ba21ff05705000d47ca8f9b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:16:30 +0100 Subject: [PATCH 01/12] Add sell signal hyperopt --- freqtrade/arguments.py | 2 +- freqtrade/optimize/default_hyperopt.py | 61 ++++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 8 +++- freqtrade/optimize/hyperopt_interface.py | 14 ++++++ user_data/hyperopts/sample_hyperopt.py | 61 ++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 19cccfe8b..b86d3502a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -272,7 +272,7 @@ class Arguments(object): '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ Default: %(default)s', - choices=['all', 'buy', 'roi', 'stoploss'], + choices=['all', 'buy', 'sell', 'roi', 'stoploss'], default='all', nargs='+', dest='spaces', diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 6139f8140..1c2d92950 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -93,6 +93,67 @@ class DefaultHyperOpts(IHyperOpt): Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by hyperopt + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use + """ + # print(params) + conditions = [] + # GUARDS AND TRENDS + if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: + conditions.append(dataframe['mfi'] > params['sell-mfi-value']) + if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: + conditions.append(dataframe['fastd'] > params['sell-fastd-value']) + if 'sell-adx-enabled' in params and params['sell-adx-enabled']: + conditions.append(dataframe['adx'] > params['sell-adx-value']) + if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: + conditions.append(dataframe['rsi'] > params['sell-rsi-value']) + + # TRIGGERS + if 'sell-trigger' in params: + if params['sell-trigger'] == 'sell-bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters + """ + return [ + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_lower', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') + ] + @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2d08fec81..674f7d3d1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -151,6 +151,9 @@ class Hyperopt(Backtesting): spaces: List[Dimension] = [] if self.has_space('buy'): spaces += self.custom_hyperopt.indicator_space() + if self.has_space('sell'): + spaces += self.custom_hyperopt.sell_indicator_space() + self.config['experimental']['use_sell_signal'] = True if self.has_space('roi'): spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): @@ -165,6 +168,9 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + if self.has_space('sell'): + self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -247,7 +253,7 @@ class Hyperopt(Backtesting): timerange=timerange ) - if self.has_space('buy'): + if self.has_space('buy') or self.has_space('sell'): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index d42206658..622de3015 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -37,6 +37,13 @@ class IHyperOpt(ABC): Create a buy strategy generator """ + @staticmethod + @abstractmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Create a sell strategy generator + """ + @staticmethod @abstractmethod def indicator_space() -> List[Dimension]: @@ -44,6 +51,13 @@ class IHyperOpt(ABC): Create an indicator space """ + @staticmethod + @abstractmethod + def sell_indicator_space() -> List[Dimension]: + """ + Create a sell indicator space + """ + @staticmethod @abstractmethod def generate_roi_table(params: Dict) -> Dict[int, float]: diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index f11236a82..ee4e4dca7 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -102,6 +102,67 @@ class SampleHyperOpts(IHyperOpt): Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by hyperopt + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use + """ + # print(params) + conditions = [] + # GUARDS AND TRENDS + if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: + conditions.append(dataframe['mfi'] > params['sell-mfi-value']) + if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: + conditions.append(dataframe['fastd'] > params['sell-fastd-value']) + if 'sell-adx-enabled' in params and params['sell-adx-enabled']: + conditions.append(dataframe['adx'] > params['sell-adx-value']) + if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: + conditions.append(dataframe['rsi'] > params['sell-rsi-value']) + + # TRIGGERS + if 'sell-trigger' in params: + if params['sell-trigger'] == 'sell-bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters + """ + return [ + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_lower', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') + ] + @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ From 5e08769366a298517a56249806dc3c61af7ab930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:27:23 +0100 Subject: [PATCH 02/12] Update hyperopt documentation with sell-stuff --- docs/hyperopt.md | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 58dc91e3a..06b90cff9 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -11,30 +11,29 @@ and still take a long time. ## Prepare Hyperopting -Before we start digging in Hyperopt, we recommend you to take a look at +Before we start digging into Hyperopt, we recommend you to take a look at an example hyperopt file located into [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) -### 1. Install a Custom Hyperopt File -This is very simple. Put your hyperopt file into the folder -`user_data/hyperopts`. +Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy. -Let assume you want a hyperopt file `awesome_hyperopt.py`:
+### 1. Install a Custom Hyperopt File + +Put your hyperopt file into the folder`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` - ### 2. Configure your Guards and Triggers -There are two places you need to change in your hyperopt file to add a -new buy hyperopt for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). + +There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: + +- Inside `indicator_space()` - the parameters hyperopt shall be optimizing. +- Inside `populate_buy_trend()` - applying the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or never buy if -current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like -"buy when EMA5 crosses over EMA10" or "buy when close price touches lower -bollinger band". +1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like @@ -45,6 +44,17 @@ If you have updated the buy strategy, ie. changed the contents of `populate_buy_trend()` method you have to update the `guards` and `triggers` hyperopts must use. +#### Sell optimization + +Similar to the buy-signal above, sell-signals can also be optimized. +Place the corresponding settings into the following methods + +- Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. +- Inside `populate_sell_trend()` - applying the parameters. + +The configuration and rules are the same than for buy signals. +To avoid naming collisions in the search-space, please prefix all sell-spaces with sell-. + ## Solving a Mystery Let's say you are curious: should you use MACD crossings or lower Bollinger @@ -125,11 +135,10 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 ``` -Use `` and `` as the names of the custom strategy -(only required for generating sells) and the custom hyperopt used. +Use `` as the name of the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. @@ -162,6 +171,7 @@ Legal values are: - `all`: optimize everything - `buy`: just search for a new buy strategy +- `sell`: just search for a new sell strategy - `roi`: just optimize the minimal profit table for your strategy - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` @@ -237,6 +247,7 @@ Once the optimized strategy has been implemented into your strategy, you should To archive the same results (number of trades, ...) than during hyperopt, please use the command line flag `--disable-max-market-positions`. This setting is the default for hyperopt for speed reasons. You can overwrite this in the configuration by setting `"position_stacking"=false` or by changing the relevant line in your hyperopt file [here](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L283). +!!! Note: Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. ## Next Step From 798ae460d8f8c91435f4e0716c50365cdf11b479 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:30:58 +0100 Subject: [PATCH 03/12] Add check if trigger is in parameters --- docs/hyperopt.md | 13 +++++++------ freqtrade/optimize/default_hyperopt.py | 21 +++++++++++---------- user_data/hyperopts/sample_hyperopt.py | 21 +++++++++++---------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 06b90cff9..c4dd76009 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -98,12 +98,13 @@ So let's write the buy strategy using these values: conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 1c2d92950..315a1e8a5 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -57,16 +57,17 @@ class DefaultHyperOpts(IHyperOpt): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index ee4e4dca7..1700af932 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -66,16 +66,17 @@ class SampleHyperOpts(IHyperOpt): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), From 2147bd88470dba08b047f76ef6c50f0281a11000 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:35:18 +0100 Subject: [PATCH 04/12] Fix problem when no experimental dict is available --- freqtrade/optimize/hyperopt.py | 3 +++ freqtrade/tests/optimize/test_hyperopt.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 674f7d3d1..ba9c186da 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -153,6 +153,9 @@ class Hyperopt(Backtesting): spaces += self.custom_hyperopt.indicator_space() if self.has_space('sell'): spaces += self.custom_hyperopt.sell_indicator_space() + # Make sure experimental is enabled + if 'experimental' not in self.config: + self.config['experimental'] = {} self.config['experimental']['use_sell_signal'] = True if self.has_space('roi'): spaces += self.custom_hyperopt.roi_space() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 53d991c09..5c6d3981a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -312,6 +312,15 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'mfi-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', + 'sell-adx-value': 0, + 'sell-fastd-value': 75, + 'sell-mfi-value': 0, + 'sell-rsi-value': 0, + 'sell-adx-enabled': False, + 'sell-fastd-enabled': True, + 'sell-mfi-enabled': False, + 'sell-rsi-enabled': False, + 'sell-trigger': 'macd_cross_signal', 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, From a0df7b9d7cfa6e5e8d5c841724e5b1a0fcc661e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:12:55 +0100 Subject: [PATCH 05/12] Use sell/buy trends from hyperopt file if available --- freqtrade/optimize/hyperopt.py | 4 ++++ freqtrade/resolvers/hyperopt_resolver.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ba9c186da..b8b8aaf56 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -170,9 +170,13 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + elif hasattr(self.custom_hyperopt, 'populate_buy_trend'): + self.advise_buy = self.custom_hyperopt.populate_buy_trend if self.has_space('sell'): self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + elif hasattr(self.custom_hyperopt, 'populate_sell_trend'): + self.advise_sell = self.custom_hyperopt.populate_sell_trend if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index eb91c0e89..6bf7fa17d 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -32,6 +32,13 @@ class HyperOptResolver(IResolver): hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + if not hasattr(self.hyperopt, 'populate_buy_trend'): + logger.warning("Custom Hyperopt does not provide populate_buy_trend. " + "Using populate_buy_trend from DefaultStrategy.") + if not hasattr(self.hyperopt, 'populate_sell_trend'): + logger.warning("Custom Hyperopt does not provide populate_sell_trend. " + "Using populate_sell_trend from DefaultStrategy.") + def _load_hyperopt( self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: """ From 167088827adbd03f7799afebe62d6f80f3c7207c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:13:15 +0100 Subject: [PATCH 06/12] include default buy/sell trends for the hyperopt strategy --- freqtrade/optimize/default_hyperopt.py | 46 ++++++++++++++++++++++---- user_data/hyperopts/sample_hyperopt.py | 46 ++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 315a1e8a5..721848d2e 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -33,6 +33,7 @@ class DefaultHyperOpts(IHyperOpt): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_upperband'] = bollinger['upper'] dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -111,21 +112,21 @@ class DefaultHyperOpts(IHyperOpt): if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: conditions.append(dataframe['fastd'] > params['sell-fastd-value']) if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] > params['sell-adx-value']) + conditions.append(dataframe['adx'] < params['sell-adx-value']) if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: conditions.append(dataframe['rsi'] > params['sell-rsi-value']) # TRIGGERS if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) if params['sell-trigger'] == 'sell-macd_cross_signal': conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] + dataframe['macdsignal'], dataframe['macd'] )) if params['sell-trigger'] == 'sell-sar_reversal': conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] + dataframe['sar'], dataframe['close'] )) dataframe.loc[ @@ -150,7 +151,7 @@ class DefaultHyperOpts(IHyperOpt): Categorical([True, False], name='sell-fastd-enabled'), Categorical([True, False], name='sell-adx-enabled'), Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_lower', + Categorical(['sell-bb_upper', 'sell-macd_cross_signal', 'sell-sar_reversal'], name='sell-trigger') ] @@ -190,3 +191,36 @@ class DefaultHyperOpts(IHyperOpt): Real(0.01, 0.07, name='roi_p2'), Real(0.01, 0.20, name='roi_p3'), ] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include buy + """ + dataframe.loc[ + ( + (dataframe['close'] < dataframe['bb_lowerband']) & + (dataframe['mfi'] < 16) & + (dataframe['adx'] > 25) & + (dataframe['rsi'] < 21) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include sell + """ + dataframe.loc[ + ( + (qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) & + (dataframe['fastd'] > 54) + ), + 'sell'] = 1 + return dataframe diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 1700af932..54f65a7e6 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -42,6 +42,7 @@ class SampleHyperOpts(IHyperOpt): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_upperband'] = bollinger['upper'] dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -120,21 +121,21 @@ class SampleHyperOpts(IHyperOpt): if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: conditions.append(dataframe['fastd'] > params['sell-fastd-value']) if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] > params['sell-adx-value']) + conditions.append(dataframe['adx'] < params['sell-adx-value']) if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: conditions.append(dataframe['rsi'] > params['sell-rsi-value']) # TRIGGERS if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) if params['sell-trigger'] == 'sell-macd_cross_signal': conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] + dataframe['macdsignal'], dataframe['macd'] )) if params['sell-trigger'] == 'sell-sar_reversal': conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] + dataframe['sar'], dataframe['close'] )) dataframe.loc[ @@ -159,7 +160,7 @@ class SampleHyperOpts(IHyperOpt): Categorical([True, False], name='sell-fastd-enabled'), Categorical([True, False], name='sell-adx-enabled'), Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_lower', + Categorical(['sell-bb_upper', 'sell-macd_cross_signal', 'sell-sar_reversal'], name='sell-trigger') ] @@ -199,3 +200,36 @@ class SampleHyperOpts(IHyperOpt): Real(0.01, 0.07, name='roi_p2'), Real(0.01, 0.20, name='roi_p3'), ] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include buy + """ + dataframe.loc[ + ( + (dataframe['close'] < dataframe['bb_lowerband']) & + (dataframe['mfi'] < 16) & + (dataframe['adx'] > 25) & + (dataframe['rsi'] < 21) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include sell + """ + dataframe.loc[ + ( + (qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) & + (dataframe['fastd'] > 54) + ), + 'sell'] = 1 + return dataframe From dd2af86a4187afc389f26c2602d0190b948ecef8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:47:38 +0100 Subject: [PATCH 07/12] pprint results --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8b8aaf56..7d6a991d2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,17 +5,18 @@ This module contains the hyperopt logic """ import logging -from argparse import Namespace +import multiprocessing import os import sys -from pathlib import Path +from argparse import Namespace from math import exp -import multiprocessing from operator import itemgetter +from pathlib import Path +from pprint import pprint from typing import Any, Dict, List -from pandas import DataFrame from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects +from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension @@ -26,7 +27,6 @@ from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.resolvers import HyperOptResolver - logger = logging.getLogger(__name__) MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -104,11 +104,11 @@ class Hyperopt(Backtesting): logger.info( 'Best result:\n%s\nwith values:\n%s', best_result['result'], - best_result['params'] + pprint(best_result['params'], indent=4) ) if 'roi_t1' in best_result['params']: logger.info('ROI table:\n%s', - self.custom_hyperopt.generate_roi_table(best_result['params'])) + pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4)) def log_results(self, results) -> None: """ From 5dd1f9b38a8f9a2ee5feecfe25e7828f3305a921 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:47:53 +0100 Subject: [PATCH 08/12] improve hyperopt docs --- docs/hyperopt.md | 58 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c4dd76009..d17464d32 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -16,6 +16,22 @@ an example hyperopt file located into [user_data/hyperopts/](https://github.com/ Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy. +### Checklist on all tasks / possibilities in hyperopt + +Depending on the space you want to optimize, only some of the below are required. + +* fill `populate_indicators` - probably a copy from your strategy +* fill `buy_strategy_generator` - for buy signal optimization +* fill `indicator_space` - for buy signal optimzation +* fill `sell_strategy_generator` - for sell signal optimization +* fill `sell_indicator_space` - for sell signal optimzation +* fill `roi_space` - for ROI optimization +* fill `generate_roi_table` - for ROI optimization (if you need more than 3 entries) +* fill `stoploss_space` - stoploss optimization +* Optional but reccomended + * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used + * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used + ### 1. Install a Custom Hyperopt File Put your hyperopt file into the folder`user_data/hyperopts`. @@ -49,11 +65,11 @@ If you have updated the buy strategy, ie. changed the contents of Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods -- Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -- Inside `populate_sell_trend()` - applying the parameters. +* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. +* Inside `populate_sell_trend()` - applying the parameters. The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with sell-. +To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. ## Solving a Mystery @@ -65,7 +81,7 @@ mystery. We will start by defining a search space: -``` +```python def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters @@ -88,7 +104,7 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: -``` +``` python def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -136,7 +152,7 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 --spaces all ``` Use `` as the name of the custom hyperopt used. @@ -144,6 +160,11 @@ Use `` as the name of the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. +The `--spaces all` flag determines that all possible parameters should be optimized. Possibilities are listed below. + +!!! Warning +When switching parameters or changing configuration options, the file `user_data/hyperopt_results.pickle` should be removed. It's used to be able to continue interrupted calculations, but does not detect changes to settings or the hyperopt file. + ### Execute Hyperopt with Different Ticker-Data Source If you would like to hyperopt parameters using an alternate ticker data that @@ -186,7 +207,11 @@ Given the following result from hyperopt: Best result: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. with values: -{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} +{ 'adx-value': 44, + 'rsi-value': 29, + 'adx-enabled': False, + 'rsi-enabled': True, + 'trigger': 'bb_lower'} ``` You should understand this result like: @@ -226,9 +251,24 @@ If you are optimizing ROI, you're result will look as follows and include a ROI Best result: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. with values: -{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower', 'roi_t1': 40, 'roi_t2': 57, 'roi_t3': 21, 'roi_p1': 0.03634636907306948, 'roi_p2': 0.055237357937802885, 'roi_p3': 0.015163796015548354, 'stoploss': -0.37996664668703606} +{ 'adx-value': 44, + 'rsi-value': 29, + 'adx-enabled': false, + 'rsi-enabled': True, + 'trigger': 'bb_lower', + 'roi_t1': 40, + 'roi_t2': 57, + 'roi_t3': 21, + 'roi_p1': 0.03634636907306948, + 'roi_p2': 0.055237357937802885, + 'roi_p3': 0.015163796015548354, + 'stoploss': -0.37996664668703606 +} ROI table: -{0: 0.10674752302642071, 21: 0.09158372701087236, 78: 0.03634636907306948, 118: 0} +{ 0: 0.10674752302642071, + 21: 0.09158372701087236, + 78: 0.03634636907306948, + 118: 0} ``` This would translate to the following ROI table: From 40b1d8f0678ce320f44ed25cbb645233be8e6336 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:57:14 +0100 Subject: [PATCH 09/12] Fix CI problems --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7d6a991d2..6930bed04 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -102,13 +102,13 @@ class Hyperopt(Backtesting): results = sorted(self.trials, key=itemgetter('loss')) best_result = results[0] logger.info( - 'Best result:\n%s\nwith values:\n%s', - best_result['result'], - pprint(best_result['params'], indent=4) + 'Best result:\n%s\nwith values:\n', + best_result['result'] ) + pprint(best_result['params'], indent=4) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', - pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4)) + logger.info('ROI table:') + pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4) def log_results(self, results) -> None: """ @@ -171,12 +171,12 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) elif hasattr(self.custom_hyperopt, 'populate_buy_trend'): - self.advise_buy = self.custom_hyperopt.populate_buy_trend + self.advise_buy = self.custom_hyperopt.populate_buy_trend # type: ignore if self.has_space('sell'): self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) elif hasattr(self.custom_hyperopt, 'populate_sell_trend'): - self.advise_sell = self.custom_hyperopt.populate_sell_trend + self.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] From 440a7ec9c2d287abf58168c5affcf88e6eda1569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 19:31:25 +0100 Subject: [PATCH 10/12] fix pytest --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 5c6d3981a..d816988a8 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -201,7 +201,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: hyperopt.start() parallel.assert_called_once() - assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text + assert 'Best result:\nfoo result\nwith values:\n\n' in caplog.text assert dumper.called From f620449bec59b4cb1df8eb5f50f458ab0a922a75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 19:38:32 +0100 Subject: [PATCH 11/12] Add test for hyperoptresolver --- freqtrade/tests/optimize/test_hyperopt.py | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index d816988a8..20baee99e 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -9,7 +9,8 @@ import pytest from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start -from freqtrade.resolvers import StrategyResolver +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts +from freqtrade.resolvers import StrategyResolver, HyperOptResolver from freqtrade.tests.conftest import log_has, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args @@ -38,6 +39,28 @@ def create_trials(mocker, hyperopt) -> None: return [{'loss': 1, 'result': 'foo', 'params': {}}] +def test_hyperoptresolver(mocker, default_conf, caplog) -> None: + + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) + hyperopts = DefaultHyperOpts + delattr(hyperopts, 'populate_buy_trend') + delattr(hyperopts, 'populate_sell_trend') + mocker.patch( + 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', + MagicMock(return_value=hyperopts) + ) + x = HyperOptResolver(default_conf, ).hyperopt + assert not hasattr(x, 'populate_buy_trend') + assert not hasattr(x, 'populate_sell_trend') + assert log_has("Custom Hyperopt does not provide populate_sell_trend. " + "Using populate_sell_trend from DefaultStrategy.", caplog.record_tuples) + assert log_has("Custom Hyperopt does not provide populate_buy_trend. " + "Using populate_buy_trend from DefaultStrategy.", caplog.record_tuples) + + def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() mocker.patch( From b5adfcf51a49da8ac65979f4af32e5f94a16a715 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 17:07:32 +0100 Subject: [PATCH 12/12] Fix documentation typo --- docs/hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index d17464d32..0c18110bd 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -28,7 +28,7 @@ Depending on the space you want to optimize, only some of the below are required * fill `roi_space` - for ROI optimization * fill `generate_roi_table` - for ROI optimization (if you need more than 3 entries) * fill `stoploss_space` - stoploss optimization -* Optional but reccomended +* Optional but recommended * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used