From f90676cfc5a249bf957b62a509ee07f6c6731e11 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Fri, 8 Nov 2019 01:55:14 +0300
Subject: [PATCH 01/13] Add trailing stoploss hyperspace

---
 freqtrade/configuration/cli_options.py   |  9 ++--
 freqtrade/optimize/hyperopt.py           | 67 +++++++++++++++++++-----
 freqtrade/optimize/hyperopt_interface.py | 16 +++++-
 tests/optimize/test_hyperopt.py          | 52 +++++++++++++-----
 4 files changed, 110 insertions(+), 34 deletions(-)

diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py
index 697e048db..e2b786ac7 100644
--- a/freqtrade/configuration/cli_options.py
+++ b/freqtrade/configuration/cli_options.py
@@ -174,12 +174,11 @@ AVAILABLE_CLI_OPTIONS = {
         default=constants.HYPEROPT_EPOCH,
     ),
     "spaces": Arg(
-        '-s', '--spaces',
-        help='Specify which parameters to hyperopt. Space-separated list. '
-        'Default: `%(default)s`.',
-        choices=['all', 'buy', 'sell', 'roi', 'stoploss'],
+        '--spaces',
+        help='Specify which parameters to hyperopt. Space-separated list.',
+        choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'],
         nargs='+',
-        default='all',
+        default='default',
     ),
     "print_all": Arg(
         '--print-all',
diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index 6ea2f5133..c5a003bd6 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -149,7 +149,7 @@ class Hyperopt:
         self.trials_file.unlink()
         return trials
 
-    def log_trials_result(self) -> None:
+    def log_trials_result(self) -> None:  # noqa: C901
         """
         Display Best hyperopt result
         """
@@ -161,14 +161,16 @@ class Hyperopt:
 
         if self.config.get('print_json'):
             result_dict: Dict = {}
+
             if self.has_space('buy') or self.has_space('sell'):
                 result_dict['params'] = {}
+
             if self.has_space('buy'):
-                result_dict['params'].update({p.name: params.get(p.name)
-                                             for p in self.hyperopt_space('buy')})
+                result_dict['params'].update(self.space_params(params, 'buy'))
+
             if self.has_space('sell'):
-                result_dict['params'].update({p.name: params.get(p.name)
-                                             for p in self.hyperopt_space('sell')})
+                result_dict['params'].update(self.space_params(params, 'sell'))
+
             if self.has_space('roi'):
                 # Convert keys in min_roi dict to strings because
                 # rapidjson cannot dump dicts with integer keys...
@@ -177,25 +179,35 @@ class Hyperopt:
                 result_dict['minimal_roi'] = OrderedDict(
                     (str(k), v) for k, v in self.custom_hyperopt.generate_roi_table(params).items()
                 )
+
             if self.has_space('stoploss'):
-                result_dict['stoploss'] = params.get('stoploss')
+                result_dict.update(self.space_params(params, 'stoploss'))
+
+            if self.has_space('trailing'):
+                result_dict.update(self.space_params(params, 'trailing'))
+
             print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
         else:
             if self.has_space('buy'):
                 print('Buy hyperspace params:')
-                pprint({p.name: params.get(p.name) for p in self.hyperopt_space('buy')},
-                       indent=4)
+                pprint(self.space_params(params, 'buy', 5), indent=4)
+
             if self.has_space('sell'):
                 print('Sell hyperspace params:')
-                pprint({p.name: params.get(p.name) for p in self.hyperopt_space('sell')},
-                       indent=4)
+                pprint(self.space_params(params, 'sell', 5), indent=4)
+
             if self.has_space('roi'):
                 print("ROI table:")
                 # Round printed values to 5 digits after the decimal point
                 pprint(round_dict(self.custom_hyperopt.generate_roi_table(params), 5), indent=4)
+
             if self.has_space('stoploss'):
-                # Also round to 5 digits after the decimal point
-                print(f"Stoploss: {round(params.get('stoploss'), 5)}")
+                print(f"Stoploss:")
+                pprint(self.space_params(params, 'stoploss', 5), indent=4)
+
+            if self.has_space('trailing'):
+                print('Trailing stop:')
+                pprint(self.space_params(params, 'trailing', 5), indent=4)
 
     def log_results(self, results) -> None:
         """
@@ -233,9 +245,13 @@ class Hyperopt:
 
     def has_space(self, space: str) -> bool:
         """
-        Tell if a space value is contained in the configuration
+        Tell if the space value is contained in the configuration
         """
-        return any(s in self.config['spaces'] for s in [space, 'all'])
+        # The 'trailing' space is not included in the 'default' set of spaces
+        if space == 'trailing':
+            return any(s in self.config['spaces'] for s in [space, 'all'])
+        else:
+            return any(s in self.config['spaces'] for s in [space, 'all', 'default'])
 
     def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
         """
@@ -245,20 +261,34 @@ class Hyperopt:
         for all hyperspaces used.
         """
         spaces: List[Dimension] = []
+
         if space == 'buy' or (space is None and self.has_space('buy')):
             logger.debug("Hyperopt has 'buy' space")
             spaces += self.custom_hyperopt.indicator_space()
+
         if space == 'sell' or (space is None and self.has_space('sell')):
             logger.debug("Hyperopt has 'sell' space")
             spaces += self.custom_hyperopt.sell_indicator_space()
+
         if space == 'roi' or (space is None and self.has_space('roi')):
             logger.debug("Hyperopt has 'roi' space")
             spaces += self.custom_hyperopt.roi_space()
+
         if space == 'stoploss' or (space is None and self.has_space('stoploss')):
             logger.debug("Hyperopt has 'stoploss' space")
             spaces += self.custom_hyperopt.stoploss_space()
+
+        if space == 'trailing' or (space is None and self.has_space('trailing')):
+            logger.debug("Hyperopt has 'trailing' space")
+            spaces += self.custom_hyperopt.trailing_space()
+
         return spaces
 
+    def space_params(self, params, space: str, r: int = None) -> Dict:
+        d = {p.name: params.get(p.name) for p in self.hyperopt_space(space)}
+        # Round floats to `r` digits after the decimal point if requested
+        return round_dict(d, r) if r else d
+
     def generate_optimizer(self, _params: Dict, iteration=None) -> Dict:
         """
         Used Optimize function. Called once per epoch to optimize whatever is configured.
@@ -281,6 +311,15 @@ class Hyperopt:
         if self.has_space('stoploss'):
             self.backtesting.strategy.stoploss = params['stoploss']
 
+        if self.has_space('trailing'):
+            self.backtesting.strategy.trailing_stop = params['trailing_stop']
+            self.backtesting.strategy.trailing_stop_positive = \
+                params['trailing_stop_positive']
+            self.backtesting.strategy.trailing_stop_positive_offset = \
+                params['trailing_stop_positive_offset']
+            self.backtesting.strategy.trailing_only_offset_is_reached = \
+                params['trailing_only_offset_is_reached']
+
         processed = load(self.tickerdata_pickle)
 
         min_date, max_date = get_timeframe(processed)
diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py
index 142f305df..e8f16d572 100644
--- a/freqtrade/optimize/hyperopt_interface.py
+++ b/freqtrade/optimize/hyperopt_interface.py
@@ -8,7 +8,7 @@ import math
 from abc import ABC
 from typing import Dict, Any, Callable, List
 
-from skopt.space import Dimension, Integer, Real
+from skopt.space import Categorical, Dimension, Integer, Real
 
 from freqtrade import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
@@ -174,6 +174,20 @@ class IHyperOpt(ABC):
             Real(-0.35, -0.02, name='stoploss'),
         ]
 
+    @staticmethod
+    def trailing_space() -> List[Dimension]:
+        """
+        Create a trailing stoploss space.
+
+        You may override it in your custom Hyperopt class.
+        """
+        return [
+            Categorical([True, False], name='trailing_stop'),
+            Real(-0.35, -0.02, name='trailing_stop_positive'),
+            Real(0.01, 0.1, name='trailing_stop_positive_offset'),
+            Categorical([True, False], name='trailing_only_offset_is_reached'),
+        ]
+
     # This is needed for proper unpickling the class attribute ticker_interval
     # which is set to the actual value by the resolver.
     # Why do I still need such shamanic mantras in modern python?
diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index 23d8a887c..e247d0134 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -26,7 +26,7 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
 
 @pytest.fixture(scope='function')
 def hyperopt(default_conf, mocker):
-    default_conf.update({'spaces': ['all']})
+    default_conf.update({'spaces': ['default']})
     patch_exchange(mocker)
     return Hyperopt(default_conf)
 
@@ -108,7 +108,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
         '--enable-position-stacking',
         '--disable-max-market-positions',
         '--epochs', '1000',
-        '--spaces', 'all',
+        '--spaces', 'default',
         '--print-all'
     ]
 
@@ -414,7 +414,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None:
     default_conf.update({'config': 'config.json.example',
                          'epochs': 1,
                          'timerange': None,
-                         'spaces': 'all',
+                         'spaces': 'default',
                          'hyperopt_jobs': 1, })
 
     hyperopt = Hyperopt(default_conf)
@@ -463,14 +463,38 @@ def test_format_results(hyperopt):
     assert result.find('Total profit 1.00000000 EUR')
 
 
-def test_has_space(hyperopt):
-    hyperopt.config.update({'spaces': ['buy', 'roi']})
-    assert hyperopt.has_space('roi')
-    assert hyperopt.has_space('buy')
-    assert not hyperopt.has_space('stoploss')
-
-    hyperopt.config.update({'spaces': ['all']})
-    assert hyperopt.has_space('buy')
+@pytest.mark.parametrize("spaces, expected_results", [
+    (['buy'],
+     {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}),
+    (['sell'],
+     {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}),
+    (['roi'],
+     {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}),
+    (['stoploss'],
+     {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}),
+    (['trailing'],
+     {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}),
+    (['buy', 'sell', 'roi', 'stoploss'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}),
+    (['buy', 'sell', 'roi', 'stoploss', 'trailing'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}),
+    (['buy', 'roi'],
+     {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}),
+    (['all'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}),
+    (['default'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}),
+    (['default', 'trailing'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}),
+    (['all', 'buy'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}),
+    (['default', 'buy'],
+     {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}),
+])
+def test_has_space(hyperopt, spaces, expected_results):
+    for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
+        hyperopt.config.update({'spaces': spaces})
+        assert hyperopt.has_space(s) == expected_results[s]
 
 
 def test_populate_indicators(hyperopt, testdatadir) -> None:
@@ -517,7 +541,7 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None:
 def test_generate_optimizer(mocker, default_conf) -> None:
     default_conf.update({'config': 'config.json.example'})
     default_conf.update({'timerange': None})
-    default_conf.update({'spaces': 'all'})
+    default_conf.update({'spaces': 'default'})
     default_conf.update({'hyperopt_min_trades': 1})
 
     trades = [
@@ -584,7 +608,7 @@ def test_clean_hyperopt(mocker, default_conf, caplog):
     default_conf.update({'config': 'config.json.example',
                          'epochs': 1,
                          'timerange': None,
-                         'spaces': 'all',
+                         'spaces': 'default',
                          'hyperopt_jobs': 1,
                          })
     mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True))
@@ -600,7 +624,7 @@ def test_continue_hyperopt(mocker, default_conf, caplog):
     default_conf.update({'config': 'config.json.example',
                          'epochs': 1,
                          'timerange': None,
-                         'spaces': 'all',
+                         'spaces': 'default',
                          'hyperopt_jobs': 1,
                          'hyperopt_continue': True
                          })

From d3a3765819f60d486e3c9e73d01dd41d7c6c3ffe Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Fri, 8 Nov 2019 03:48:08 +0300
Subject: [PATCH 02/13] Fix test

---
 tests/optimize/test_hyperopt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index e247d0134..aaaeb8795 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -668,7 +668,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     parallel.assert_called_once()
 
     out, err = capsys.readouterr()
-    assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null}' in out  # noqa: E501
+    assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null,"trailing_stop_positive":null,"trailing_stop_positive_offset":null,"trailing_only_offset_is_reached":null}' in out  # noqa: E501
     assert dumper.called
     # Should be called twice, once for tickerdata, once to save evaluations
     assert dumper.call_count == 2

From 31ab32f0b965f112c796112f830d0c3e550ddc7d Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Fri, 8 Nov 2019 12:47:28 +0300
Subject: [PATCH 03/13] Always set trailing_stop=True with 'trailing'
 hyperspace

---
 freqtrade/optimize/hyperopt.py           | 3 +--
 freqtrade/optimize/hyperopt_interface.py | 9 ++++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index c5a003bd6..e5e027d46 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -313,8 +313,7 @@ class Hyperopt:
 
         if self.has_space('trailing'):
             self.backtesting.strategy.trailing_stop = params['trailing_stop']
-            self.backtesting.strategy.trailing_stop_positive = \
-                params['trailing_stop_positive']
+            self.backtesting.strategy.trailing_stop_positive = params['trailing_stop_positive']
             self.backtesting.strategy.trailing_stop_positive_offset = \
                 params['trailing_stop_positive_offset']
             self.backtesting.strategy.trailing_only_offset_is_reached = \
diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py
index e8f16d572..b4cfdcaf3 100644
--- a/freqtrade/optimize/hyperopt_interface.py
+++ b/freqtrade/optimize/hyperopt_interface.py
@@ -182,7 +182,14 @@ class IHyperOpt(ABC):
         You may override it in your custom Hyperopt class.
         """
         return [
-            Categorical([True, False], name='trailing_stop'),
+            # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace
+            # is used. Otherwise hyperopt will vary other parameters that won't have effect if
+            # trailing_stop is set False.
+            # This parameter is included into the hyperspace dimensions rather than assigning
+            # it explicitly in the code in order to have it printed in the results along with
+            # other 'trailing' hyperspace parameters.
+            Categorical([True], name='trailing_stop'),
+
             Real(-0.35, -0.02, name='trailing_stop_positive'),
             Real(0.01, 0.1, name='trailing_stop_positive_offset'),
             Categorical([True, False], name='trailing_only_offset_is_reached'),

From ab194c7d7572a045399d39603a68f2f683e5ebfc Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Wed, 13 Nov 2019 23:09:05 +0300
Subject: [PATCH 04/13] Add test

---
 tests/optimize/test_hyperopt.py | 38 +++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index aaaeb8795..123fbf995 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -674,6 +674,44 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
     assert dumper.call_count == 2
 
 
+def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None:
+    dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
+                 MagicMock(return_value=(MagicMock(), None)))
+    mocker.patch(
+        'freqtrade.optimize.hyperopt.get_timeframe',
+        MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
+    )
+
+    parallel = mocker.patch(
+        'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
+        MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
+    )
+    patch_exchange(mocker)
+
+    default_conf.update({'config': 'config.json.example',
+                         'epochs': 1,
+                         'timerange': None,
+                         'spaces': 'default',
+                         'hyperopt_jobs': 1,
+                         'print_json': True,
+                         })
+
+    hyperopt = Hyperopt(default_conf)
+    hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock()
+    hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
+
+    hyperopt.start()
+
+    parallel.assert_called_once()
+
+    out, err = capsys.readouterr()
+    assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null}' in out  # noqa: E501
+    assert dumper.called
+    # Should be called twice, once for tickerdata, once to save evaluations
+    assert dumper.call_count == 2
+
+
 def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None:
     dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',

From f9a92c28797f90fb1fc9b51d863bb2135341a0e3 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Wed, 13 Nov 2019 23:32:37 +0300
Subject: [PATCH 05/13] Adjust test

---
 tests/optimize/test_hyperopt.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index 123fbf995..9a168d3e2 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -541,7 +541,7 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None:
 def test_generate_optimizer(mocker, default_conf) -> None:
     default_conf.update({'config': 'config.json.example'})
     default_conf.update({'timerange': None})
-    default_conf.update({'spaces': 'default'})
+    default_conf.update({'spaces': 'all'})
     default_conf.update({'hyperopt_min_trades': 1})
 
     trades = [
@@ -587,6 +587,10 @@ def test_generate_optimizer(mocker, default_conf) -> None:
         'roi_p2': 0.01,
         'roi_p3': 0.1,
         'stoploss': -0.4,
+        'trailing_stop': True,
+        'trailing_stop_positive': 0.02,
+        'trailing_stop_positive_offset': 0.1,
+        'trailing_only_offset_is_reached': False,
     }
     response_expected = {
         'loss': 1.9840569076926293,

From 175591e524313e3fba2b89eb111cd89e4833581b Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sat, 23 Nov 2019 04:03:47 +0300
Subject: [PATCH 06/13] Fix test

---
 tests/optimize/test_hyperopt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py
index 6b0f5eafa..6ff7968c5 100644
--- a/tests/optimize/test_hyperopt.py
+++ b/tests/optimize/test_hyperopt.py
@@ -718,6 +718,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
     patch_exchange(mocker)
 
     default_conf.update({'config': 'config.json.example',
+                         'hyperopt': 'DefaultHyperOpt',
                          'epochs': 1,
                          'timerange': None,
                          'spaces': 'default',

From fffd47e3d86cba530cf18bf1db758df8ea0bcb91 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sun, 1 Dec 2019 01:28:26 +0300
Subject: [PATCH 07/13] Add description of trailing space into docs

---
 docs/hyperopt.md | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index 6c1505e75..15ef26c99 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -381,12 +381,6 @@ Best result:
 
     44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
 
-Buy hyperspace params:
-{   'adx-value': 44,
-    'rsi-value': 29,
-    'adx-enabled': False,
-    'rsi-enabled': True,
-    'trigger': 'bb_lower'}
 ROI table:
 {   0: 0.10674,
     21: 0.09158,
@@ -410,7 +404,7 @@ As stated in the comment, you can also use it as the value of the `minimal_roi`
 
 #### Default ROI Search Space
 
-If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values can vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point):
+If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point):
 
 | # step | 1m |  | 5m |  | 1h |  | 1d |  |
 |---|---|---|---|---|---|---|---|---|
@@ -454,12 +448,46 @@ As stated in the comment, you can also use it as the value of the `stoploss` set
 
 #### Default Stoploss Search Space
 
-If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace can vary in the range -0.35...-0.02, which is sufficient in most cases.
+If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimization hyperspace for you. By default, the stoploss values in that hyperspace vary in the range -0.35...-0.02, which is sufficient in most cases.
 
 If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default.
 
 Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py).
 
+### Understand Hyperopt Trailing Stop results
+
+If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters:
+
+```
+Best result:
+
+    45/100:    606 trades. Avg profit  1.04%. Total profit  0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161
+
+Trailing stop:
+{   'trailing_only_offset_is_reached': True,
+    'trailing_stop': True,
+    'trailing_stop_positive': -0.02005,
+    'trailing_stop_positive_offset': 0.05726}
+```
+
+In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy:
+
+```
+    # Trailing stop
+    # These attributes will be overridden if the config file contains corresponding values.
+    trailing_stop = True
+    trailing_stop_positive = -0.02005
+    trailing_stop_positive_offset = 0.05726
+    trailing_only_offset_is_reached = True
+```
+As stated in the comment, you can also use it as the values of the corresponding settings in the configuration file.
+
+#### Default Trailing Stop Search Space
+
+If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges -0.35...-0.02 and 0.01...0.1 correspondingly, which is sufficient in most cases.
+
+Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py).
+
 ### Validate backtesting results
 
 Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected.

From a88bfa8ded672deda3d68e7cc45ceb0e69289d44 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 1 Dec 2019 02:27:17 +0300
Subject: [PATCH 08/13] Fix: trailing_stop_positive should be positive

---
 docs/hyperopt.md                         | 10 +++++-----
 freqtrade/optimize/hyperopt_interface.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index 15ef26c99..d9aa3c7a0 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -466,8 +466,8 @@ Best result:
 Trailing stop:
 {   'trailing_only_offset_is_reached': True,
     'trailing_stop': True,
-    'trailing_stop_positive': -0.02005,
-    'trailing_stop_positive_offset': 0.05726}
+    'trailing_stop_positive': 0.02001,
+    'trailing_stop_positive_offset': 0.06038}
 ```
 
 In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy:
@@ -476,15 +476,15 @@ In order to use these best trailing stop parameters found by Hyperopt in backtes
     # Trailing stop
     # These attributes will be overridden if the config file contains corresponding values.
     trailing_stop = True
-    trailing_stop_positive = -0.02005
-    trailing_stop_positive_offset = 0.05726
+    trailing_stop_positive = 0.02001
+    trailing_stop_positive_offset = 0.06038
     trailing_only_offset_is_reached = True
 ```
 As stated in the comment, you can also use it as the values of the corresponding settings in the configuration file.
 
 #### Default Trailing Stop Search Space
 
-If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges -0.35...-0.02 and 0.01...0.1 correspondingly, which is sufficient in most cases.
+If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges 0.02...0.35 and 0.01...0.1 correspondingly, which is sufficient in most cases.
 
 Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py).
 
diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py
index 0400a8470..958fb8d66 100644
--- a/freqtrade/optimize/hyperopt_interface.py
+++ b/freqtrade/optimize/hyperopt_interface.py
@@ -190,7 +190,7 @@ class IHyperOpt(ABC):
             # other 'trailing' hyperspace parameters.
             Categorical([True], name='trailing_stop'),
 
-            Real(-0.35, -0.02, name='trailing_stop_positive'),
+            Real(0.02, 0.35, name='trailing_stop_positive'),
             Real(0.01, 0.1, name='trailing_stop_positive_offset'),
             Categorical([True, False], name='trailing_only_offset_is_reached'),
         ]

From f862b4d0f0c7ed1640e0429718e267e2aa5149e2 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sun, 1 Dec 2019 02:50:44 +0300
Subject: [PATCH 09/13] Add description for 'default' space

---
 docs/hyperopt.md | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index d9aa3c7a0..ba38b9308 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -43,8 +43,9 @@ Optional - can also be loaded from a strategy:
 Rarely you may also need to override:
 
 * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
-* `generate_roi_table` - for custom ROI optimization (if you need more than 4 entries in the ROI table)
+* `generate_roi_table` - for custom ROI optimization (if you need more than 4 entries in the ROI table or the ranges of the values in the ROI table that differ from default)
 * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default)
+* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stopl parameters in the optimization hyperspace that differ from default)
 
 ### 1. Install a Custom Hyperopt File
 
@@ -250,10 +251,10 @@ freqtrade hyperopt --config config.json --hyperopt <hyperoptname> -e 5000 --spac
 
 Use  `<hyperoptname>` as the name of the custom hyperopt used.
 
-The `-e` flag will set how many evaluations hyperopt will do. We recommend
+The `-e` option will set how many evaluations hyperopt will do. We recommend
 running at least several thousand evaluations.
 
-The `--spaces all` flag determines that all possible parameters should be optimized. Possibilities are listed below.
+The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below.
 
 !!! Note
     By default, hyperopt will erase previous results and start from scratch. Continuation can be archived by using `--continue`.
@@ -286,7 +287,7 @@ freqtrade hyperopt --strategy SampleStrategy --customhyperopt SampleHyperopt
 
 ### Running Hyperopt with Smaller Search Space
 
-Use the `--spaces` argument to limit the search space used by hyperopt.
+Use the `--spaces` option to limit the search space used by hyperopt.
 Letting Hyperopt optimize everything is a huuuuge search space. Often it
 might make more sense to start by just searching for initial buy algorithm.
 Or maybe you just want to optimize your stoploss or roi table for that awesome
@@ -299,8 +300,12 @@ Legal values are:
 * `sell`: just search for a new sell strategy
 * `roi`: just optimize the minimal profit table for your strategy
 * `stoploss`: search for the best stoploss value
+* `trailing`: search for the best trailing stop values
+* `default`: `all` except `trailing`
 * space-separated list of any of the above values for example `--spaces roi stoploss`
 
+The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy.
+
 ### Position stacking and disabling max market positions
 
 In some situations, you may need to run Hyperopt (and Backtesting) with the 
@@ -374,7 +379,7 @@ You can use the `--print-all` command line option if you would like to see all r
 
 ### Understand Hyperopt ROI results
 
-If you are optimizing ROI (i.e. if optimization search-space contains 'all' or 'roi'), your result will look as follows and include a ROI table:
+If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:
 
 ```
 Best result:
@@ -421,7 +426,7 @@ Override the `roi_space()` method if you need components of the ROI tables to va
 
 ### Understand Hyperopt Stoploss results
 
-If you are optimizing stoploss values (i.e. if optimization search-space contains 'all' or 'stoploss'), your result will look as follows and include stoploss:
+If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss:
 
 ```
 Best result:

From 26a7af85eaae69bce974c6ac97837bb248384a05 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 1 Dec 2019 03:31:03 +0300
Subject: [PATCH 10/13] Add trailing_space() into AdvancedSampleHyperOpt

---
 .../templates/sample_hyperopt_advanced.py     | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py
index 5634c21ea..b4bbee3fb 100644
--- a/freqtrade/templates/sample_hyperopt_advanced.py
+++ b/freqtrade/templates/sample_hyperopt_advanced.py
@@ -233,6 +233,27 @@ class AdvancedSampleHyperOpt(IHyperOpt):
             Real(-0.5, -0.02, name='stoploss'),
         ]
 
+    @staticmethod
+    def trailing_space() -> List[Dimension]:
+        """
+        Create a trailing stoploss space.
+
+        You may override it in your custom Hyperopt class.
+        """
+        return [
+            # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace
+            # is used. Otherwise hyperopt will vary other parameters that won't have effect if
+            # trailing_stop is set False.
+            # This parameter is included into the hyperspace dimensions rather than assigning
+            # it explicitly in the code in order to have it printed in the results along with
+            # other 'trailing' hyperspace parameters.
+            Categorical([True], name='trailing_stop'),
+
+            Real(0.02, 0.35, name='trailing_stop_positive'),
+            Real(0.01, 0.1, name='trailing_stop_positive_offset'),
+            Categorical([True, False], name='trailing_only_offset_is_reached'),
+        ]
+
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
         Based on TA indicators.

From f42ce8fc2a27e6118cd5a8be2aff3e67ee5fa635 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sun, 1 Dec 2019 13:07:17 +0300
Subject: [PATCH 11/13] Fix typo in the docs

---
 docs/hyperopt.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index 640ea6b74..3a5589d40 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -48,7 +48,7 @@ Rarely you may also need to override:
 * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
 * `generate_roi_table` - for custom ROI optimization (if you need more than 4 entries in the ROI table or the ranges of the values in the ROI table that differ from default)
 * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default)
-* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stopl parameters in the optimization hyperspace that differ from default)
+* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
 
 ### 1. Install a Custom Hyperopt File
 

From 32c9b5f415b7c471bfb3d7ba2e746dcc097a1902 Mon Sep 17 00:00:00 2001
From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com>
Date: Sun, 1 Dec 2019 13:13:41 +0300
Subject: [PATCH 12/13] Description for generate_roi_table reformulated
 slightly

---
 docs/hyperopt.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
index 3a5589d40..c4db6da87 100644
--- a/docs/hyperopt.md
+++ b/docs/hyperopt.md
@@ -46,7 +46,7 @@ Optional - can also be loaded from a strategy:
 Rarely you may also need to override:
 
 * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
-* `generate_roi_table` - for custom ROI optimization (if you need more than 4 entries in the ROI table or the ranges of the values in the ROI table that differ from default)
+* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps)
 * `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default)
 * `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
 

From 668d42447fa404835e874e6113f846ca51b72436 Mon Sep 17 00:00:00 2001
From: hroff-1902 <hroff_raeyd9tzpttf@mail.ru>
Date: Sun, 1 Dec 2019 16:15:00 +0300
Subject: [PATCH 13/13] Refactor log_trials_result()

---
 freqtrade/optimize/hyperopt.py | 69 ++++++++++++++--------------------
 1 file changed, 29 insertions(+), 40 deletions(-)

diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py
index d79e1336e..9c585ed72 100644
--- a/freqtrade/optimize/hyperopt.py
+++ b/freqtrade/optimize/hyperopt.py
@@ -156,7 +156,7 @@ class Hyperopt:
         self.trials_file.unlink()
         return trials
 
-    def log_trials_result(self) -> None:  # noqa: C901
+    def log_trials_result(self) -> None:
         """
         Display Best hyperopt result
         """
@@ -170,57 +170,43 @@ class Hyperopt:
         best_result = results[0]
         params = best_result['params']
         log_str = self.format_results_logstring(best_result)
+
         print(f"\nBest result:\n\n{log_str}\n")
 
         if self.config.get('print_json'):
             result_dict: Dict = {}
+            for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
+                self._params_update_for_json(result_dict, params, s)
+            print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
 
-            if self.has_space('buy') or self.has_space('sell'):
-                result_dict['params'] = {}
+        else:
+            self._params_pretty_print(params, 'buy', "Buy hyperspace params:")
+            self._params_pretty_print(params, 'sell', "Sell hyperspace params:")
+            self._params_pretty_print(params, 'roi', "ROI table:")
+            self._params_pretty_print(params, 'stoploss', "Stoploss:")
+            self._params_pretty_print(params, 'trailing', "Trailing stop:")
 
-            if self.has_space('buy'):
-                result_dict['params'].update(self.space_params(params, 'buy'))
-
-            if self.has_space('sell'):
-                result_dict['params'].update(self.space_params(params, 'sell'))
-
-            if self.has_space('roi'):
+    def _params_update_for_json(self, result_dict, params, space: str):
+        if self.has_space(space):
+            space_params = self.space_params(params, space)
+            if space in ['buy', 'sell']:
+                result_dict.setdefault('params', {}).update(space_params)
+            elif space == 'roi':
                 # Convert keys in min_roi dict to strings because
                 # rapidjson cannot dump dicts with integer keys...
                 # OrderedDict is used to keep the numeric order of the items
                 # in the dict.
                 result_dict['minimal_roi'] = OrderedDict(
-                    (str(k), v) for k, v in self.custom_hyperopt.generate_roi_table(params).items()
+                    (str(k), v) for k, v in space_params.items()
                 )
+            else:  # 'stoploss', 'trailing'
+                result_dict.update(space_params)
 
-            if self.has_space('stoploss'):
-                result_dict.update(self.space_params(params, 'stoploss'))
-
-            if self.has_space('trailing'):
-                result_dict.update(self.space_params(params, 'trailing'))
-
-            print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
-        else:
-            if self.has_space('buy'):
-                print('Buy hyperspace params:')
-                pprint(self.space_params(params, 'buy', 5), indent=4)
-
-            if self.has_space('sell'):
-                print('Sell hyperspace params:')
-                pprint(self.space_params(params, 'sell', 5), indent=4)
-
-            if self.has_space('roi'):
-                print("ROI table:")
-                # Round printed values to 5 digits after the decimal point
-                pprint(round_dict(self.custom_hyperopt.generate_roi_table(params), 5), indent=4)
-
-            if self.has_space('stoploss'):
-                print(f"Stoploss:")
-                pprint(self.space_params(params, 'stoploss', 5), indent=4)
-
-            if self.has_space('trailing'):
-                print('Trailing stop:')
-                pprint(self.space_params(params, 'trailing', 5), indent=4)
+    def _params_pretty_print(self, params, space: str, header: str):
+        if self.has_space(space):
+            space_params = self.space_params(params, space, 5)
+            print(header)
+            pprint(space_params, indent=4)
 
     def is_best(self, results) -> bool:
         return results['loss'] < self.current_best_loss
@@ -302,7 +288,10 @@ class Hyperopt:
         return spaces
 
     def space_params(self, params, space: str, r: int = None) -> Dict:
-        d = {p.name: params.get(p.name) for p in self.hyperopt_space(space)}
+        if space == 'roi':
+            d = self.custom_hyperopt.generate_roi_table(params)
+        else:
+            d = {p.name: params.get(p.name) for p in self.hyperopt_space(space)}
         # Round floats to `r` digits after the decimal point if requested
         return round_dict(d, r) if r else d