From 750c780293facfecd90bbb864e783fb49070a134 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 16:37:19 +0200 Subject: [PATCH 01/48] Support loading parameters from json file --- freqtrade/strategy/hyper.py | 29 ++++++++++++++++++++++++++--- tests/optimize/test_hyperopt.py | 2 ++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 21a806202..3ae500847 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -5,8 +5,10 @@ This module defines a base class for auto-hyperoptable strategies. import logging from abc import ABC, abstractmethod from contextlib import suppress +from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union +from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.optimize.hyperopt_tools import HyperoptTools @@ -305,10 +307,31 @@ class HyperStrategyMixin(object): """ Load Hyperoptable parameters """ - self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt) - self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt) + params = self.load_params_from_file() + params = params.get('params', {}) + buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', None)) + sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', None)) - def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None: + self._load_params(buy_params, 'buy', hyperopt) + self._load_params(sell_params, 'sell', hyperopt) + + def load_params_from_file(self) -> Dict: + filename_str = getattr(self, '__file__', '') + if not filename_str: + return {} + filename = Path(filename_str).with_suffix('.json') + + if filename.is_file(): + logger.info(f"Loading parameters from file {filename}") + params = json_load(filename.open('r')) + if params.get('strategy_name') != self.get_strategy_name(): + raise OperationalException('Invalid parameter file provided') + return params + logger.info("Found no parameter file.") + + return {} + + def _load_params(self, params: Dict, space: str, hyperopt: bool = False) -> None: """ Set optimizable parameter values. :param params: Dictionary with new parameter values. diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 10e99395d..c4cea638f 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -686,6 +686,8 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: def test_clean_hyperopt(mocker, hyperopt_conf, caplog): patch_exchange(mocker) + mocker.patch("freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file", + MagicMock(return_value={})) mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) h = Hyperopt(hyperopt_conf) From 2bf17f71e7e6984a9cac61050272e98327341872 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 16:49:28 +0200 Subject: [PATCH 02/48] Dump parameters from hyperopt-show --- freqtrade/commands/hyperopt_commands.py | 8 ++++++- freqtrade/optimize/hyperopt_tools.py | 32 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 19337b407..078781114 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -129,9 +129,15 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: metrics = val['results_metrics'] if 'strategy_name' in metrics: - show_backtest_result(metrics['strategy_name'], metrics, + strategy_name = metrics['strategy_name'] + show_backtest_result(strategy_name, metrics, metrics['stake_currency']) + # Export parameters ... + # TODO: make this optional? otherwise it'll overwrite previous parameters ... + fn = HyperoptTools.get_strategy_filename(config, strategy_name) + HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 9eee42a8d..0d0f07c8e 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -1,6 +1,7 @@ import io import logging +from copy import deepcopy from pathlib import Path from typing import Any, Dict, List @@ -9,8 +10,9 @@ import tabulate from colorama import Fore, Style from pandas import isna, json_normalize +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException -from freqtrade.misc import round_coin_value, round_dict +from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict logger = logging.getLogger(__name__) @@ -18,6 +20,34 @@ logger = logging.getLogger(__name__) class HyperoptTools(): + @staticmethod + def get_strategy_filename(config: Dict, strategy_name: str) -> Path: + """ + Get Strategy-location (filename) from strategy_name + """ + from freqtrade.resolvers.strategy_resolver import StrategyResolver + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) + strategy_objs = StrategyResolver.search_all_objects(directory, False) + strategy = [s for s in strategy_objs if s['name'] == strategy_name] + if strategy: + strategy = strategy[0] + + return Path(strategy['location']) + + @staticmethod + def export_params(params, strategy_name: str, filename: Path): + """ + Generate files + """ + final_params = deepcopy(params['params_not_optimized']) + final_params = deep_merge_dicts(params['params_details'], final_params) + final_params = { + 'strategy_name': strategy_name, + 'params': final_params + } + logger.info(f"Dumping parameters to {filename}") + rapidjson.dump(final_params, filename.open('w'), indent=2) + @staticmethod def has_space(config: Dict[str, Any], space: str) -> bool: """ From 8cdd1e3aef53aee4bf56b73692c34085ceeaafa1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 16:56:36 +0200 Subject: [PATCH 03/48] Fix some type errors --- freqtrade/commands/hyperopt_commands.py | 5 ++++- freqtrade/optimize/hyperopt_tools.py | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 078781114..e5c9241f0 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -136,7 +136,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: # Export parameters ... # TODO: make this optional? otherwise it'll overwrite previous parameters ... fn = HyperoptTools.get_strategy_filename(config, strategy_name) - HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + if fn: + HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + else: + logger.warn("Strategy not found, not exporting parameter file.") HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 0d0f07c8e..dcffab8b2 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -3,7 +3,7 @@ import io import logging from copy import deepcopy from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import rapidjson import tabulate @@ -21,18 +21,19 @@ logger = logging.getLogger(__name__) class HyperoptTools(): @staticmethod - def get_strategy_filename(config: Dict, strategy_name: str) -> Path: + def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]: """ Get Strategy-location (filename) from strategy_name """ from freqtrade.resolvers.strategy_resolver import StrategyResolver directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) strategy_objs = StrategyResolver.search_all_objects(directory, False) - strategy = [s for s in strategy_objs if s['name'] == strategy_name] - if strategy: - strategy = strategy[0] + strategies = [s for s in strategy_objs if s['name'] == strategy_name] + if strategies: + strategy = strategies[0] return Path(strategy['location']) + return None @staticmethod def export_params(params, strategy_name: str, filename: Path): From 2310deec53ad6f06fea706da9a63326925a4b433 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 20:42:54 +0200 Subject: [PATCH 04/48] Update name to get non-optimized parameters --- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/strategy/hyper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c2b2b93cb..4d2924bf4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -310,7 +310,7 @@ class Hyperopt: results_explanation = HyperoptTools.format_results_explanation_string( strat_stats, self.config['stake_currency']) - not_optimized = self.backtesting.strategy.get_params_dict() + not_optimized = self.backtesting.strategy.get_no_optimize_params() trade_count = strat_stats['total_trades'] total_profit = strat_stats['profit_total'] diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 3ae500847..0ced4dfb1 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -358,7 +358,7 @@ class HyperStrategyMixin(object): else: logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}') - def get_params_dict(self): + def get_no_optimize_params(self): """ Returns list of Parameters that are not part of the current optimize job """ From 34e6ce431f60f6cca0406546a4967d75f10b0e5f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 20:45:06 +0200 Subject: [PATCH 05/48] Print non-optimized parameters (also stop / roi) --- freqtrade/optimize/hyperopt.py | 22 +++++++++++++++++++++- freqtrade/optimize/hyperopt_tools.py | 28 ++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4d2924bf4..c23884bcd 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -23,7 +23,7 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange -from freqtrade.misc import file_dump_json, plural +from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto @@ -201,6 +201,25 @@ class Hyperopt: return result + def _get_no_optimize_details(self) -> Dict[str, Any]: + """ + Get non-optimized parameters + """ + result: Dict[str, Any] = {} + strategy = self.backtesting.strategy + if not HyperoptTools.has_space(self.config, 'roi'): + result['roi'] = strategy.minimal_roi + if not HyperoptTools.has_space(self.config, 'stoploss'): + result['stoploss'] = strategy.stoploss + if not HyperoptTools.has_space(self.config, 'trailing'): + result['trailing'] = { + 'trailing_stop': strategy.trailing_stop, + 'trailing_stop_positive': strategy.trailing_stop_positive, + 'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset, + 'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached, + } + return result + def print_results(self, results) -> None: """ Log results if it is better than any previous evaluation @@ -311,6 +330,7 @@ class Hyperopt: strat_stats, self.config['stake_currency']) not_optimized = self.backtesting.strategy.get_no_optimize_params() + not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details()) trade_count = strat_stats['total_trades'] total_profit = strat_stats['profit_total'] diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index dcffab8b2..0d17a5d13 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -130,9 +130,9 @@ class HyperoptTools(): non_optimized) HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:", non_optimized) - HyperoptTools._params_pretty_print(params, 'roi', "ROI table:") - HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:") - HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:") + HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized) + HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized) + HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized) @staticmethod def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: @@ -159,19 +159,31 @@ class HyperoptTools(): if space in params or space in non_optimized: space_params = HyperoptTools._space_params(params, space, 5) result = f"\n# {header}\n" - if space == 'stoploss': - result += f"stoploss = {space_params.get('stoploss')}" - elif space == 'roi': + if space == "stoploss": + opt = True + if not space_params: + space_params = HyperoptTools._space_params(params, space, 5) + opt = False + result += (f"stoploss = {space_params.get('stoploss')}" + f"{' # value loaded from strategy' if not opt else ''}") + + elif space == "roi": minimal_roi_result = rapidjson.dumps({ str(k): v for k, v in space_params.items() }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" - elif space == 'trailing': + elif space == "trailing": + opt = True + if not space_params: + # Not optimized ... + space_params = HyperoptTools._space_params(non_optimized, space, 5) + opt = False for k, v in space_params.items(): - result += f'{k} = {v}\n' + result += f"{k} = {v}{' # value loaded from strategy' if not opt else ''}\n" else: + # Buy / sell parameters no_params = HyperoptTools._space_params(non_optimized, space, 5) result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}" From e97de4643fe1f0389523cdacf252c57029879d64 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 21:06:15 +0200 Subject: [PATCH 06/48] Move tests to hyperopttools test file --- tests/optimize/test_hyperopt.py | 137 ------------------------- tests/optimize/test_hyperopttools.py | 146 +++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 137 deletions(-) create mode 100644 tests/optimize/test_hyperopttools.py diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index c4cea638f..91d9f5496 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,9 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import logging -import re from datetime import datetime from pathlib import Path -from typing import Dict, List from unittest.mock import ANY, MagicMock import pandas as pd @@ -28,12 +25,6 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, from .hyperopts.default_hyperopt import DefaultHyperOpt -# Functions for recurrent object patching -def create_results() -> List[Dict]: - - return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] - - def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -303,52 +294,6 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_results_saves_epochs(mocker, hyperopt, tmpdir, caplog) -> None: - # Test writing to temp dir and reading again - epochs = create_results() - hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') - - caplog.set_level(logging.DEBUG) - - for epoch in epochs: - hyperopt._save_result(epoch) - assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog) - - hyperopt._save_result(epochs[0]) - assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) - - hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) - assert len(hyperopt_epochs) == 2 - - -def test_load_previous_results(testdatadir, caplog) -> None: - - results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading pickled epochs from .*", caplog) - - caplog.clear() - - # Modern version - results_file = testdatadir / 'strategy_SampleStrategy.fthypt' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading epochs from .*", caplog) - - -def test_load_previous_results2(mocker, testdatadir, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', - return_value=[{'asdf': '222'}]) - results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): - HyperoptTools.load_previous_results(results_file) - - def test_roi_table_generation(hyperopt) -> None: params = { 'roi_t1': 5, @@ -467,40 +412,6 @@ def test_hyperopt_format_results(hyperopt): assert '0:50:00 min' in result -@pytest.mark.parametrize("spaces, expected_results", [ - (['buy'], - {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}), - (['sell'], - {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}), - (['roi'], - {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), - (['stoploss'], - {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}), - (['trailing'], - {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}), - (['buy', 'sell', 'roi', 'stoploss'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), - (['buy', 'sell', 'roi', 'stoploss', 'trailing'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['buy', 'roi'], - {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), - (['all'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['default'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), - (['default', 'trailing'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['all', 'buy'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['default', 'buy'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), -]) -def test_has_space(hyperopt_conf, spaces, expected_results): - for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - hyperopt_conf.update({'spaces': spaces}) - assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] - - def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) @@ -1070,42 +981,6 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.start() -def test_show_epoch_details(capsys): - test_result = { - 'params_details': { - 'trailing': { - 'trailing_stop': True, - 'trailing_stop_positive': 0.02, - 'trailing_stop_positive_offset': 0.04, - 'trailing_only_offset_is_reached': True - }, - 'roi': { - 0: 0.18, - 90: 0.14, - 225: 0.05, - 430: 0}, - }, - 'results_explanation': 'foo result', - 'is_initial_point': False, - 'total_profit': 0, - 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) - 'is_best': True - } - - HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) - captured = capsys.readouterr() - assert '# Trailing stop:' in captured.out - # re.match(r"Pairs for .*", captured.out) - assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE) - - assert '# ROI table:' in captured.out - assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) - assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) - - def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -1147,15 +1022,3 @@ def test_SKDecimal(): assert space.transform([1.5, 1.6]) == [150, 160] -def test___pprint(): - params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} - non_params = {'buy_notoptimied': 55} - - x = HyperoptTools._pprint(params, non_params) - assert x == """{ - "buy_std": 1.2, - "buy_rsi": 31, - "buy_enable": True, - "buy_what": "asdf", - "buy_notoptimied": 55, # value loaded from strategy -}""" diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py new file mode 100644 index 000000000..94216f2f7 --- /dev/null +++ b/tests/optimize/test_hyperopttools.py @@ -0,0 +1,146 @@ +import logging +import re +from pathlib import Path +from typing import Dict, List + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.hyperopt_tools import HyperoptTools +from tests.conftest import log_has, log_has_re + + +# Functions for recurrent object patching +def create_results() -> List[Dict]: + + return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] + + +def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None: + # Test writing to temp dir and reading again + epochs = create_results() + hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') + + caplog.set_level(logging.DEBUG) + + for epoch in epochs: + hyperopt._save_result(epoch) + assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog) + + hyperopt._save_result(epochs[0]) + assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) + + hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) + assert len(hyperopt_epochs) == 2 + + +def test_load_previous_results(testdatadir, caplog) -> None: + + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading pickled epochs from .*", caplog) + + caplog.clear() + + # Modern version + results_file = testdatadir / 'strategy_SampleStrategy.fthypt' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading epochs from .*", caplog) + + +def test_load_previous_results2(mocker, testdatadir, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', + return_value=[{'asdf': '222'}]) + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): + HyperoptTools.load_previous_results(results_file) + + +@pytest.mark.parametrize("spaces, expected_results", [ + (['buy'], + {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}), + (['sell'], + {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}), + (['roi'], + {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['stoploss'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}), + (['trailing'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}), + (['buy', 'sell', 'roi', 'stoploss'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['buy', 'sell', 'roi', 'stoploss', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['buy', 'roi'], + {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['all'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['default', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['all', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), +]) +def test_has_space(hyperopt_conf, spaces, expected_results): + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: + hyperopt_conf.update({'spaces': spaces}) + assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] + + +def test_show_epoch_details(capsys): + test_result = { + 'params_details': { + 'trailing': { + 'trailing_stop': True, + 'trailing_stop_positive': 0.02, + 'trailing_stop_positive_offset': 0.04, + 'trailing_only_offset_is_reached': True + }, + 'roi': { + 0: 0.18, + 90: 0.14, + 225: 0.05, + 430: 0}, + }, + 'results_explanation': 'foo result', + 'is_initial_point': False, + 'total_profit': 0, + 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) + 'is_best': True + } + + HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) + captured = capsys.readouterr() + assert '# Trailing stop:' in captured.out + # re.match(r"Pairs for .*", captured.out) + assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE) + + assert '# ROI table:' in captured.out + assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) + assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) + + +def test___pprint(): + params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} + non_params = {'buy_notoptimied': 55} + + x = HyperoptTools._pprint(params, non_params) + assert x == """{ + "buy_std": 1.2, + "buy_rsi": 31, + "buy_enable": True, + "buy_what": "asdf", + "buy_notoptimied": 55, # value loaded from strategy +}""" From ef14359d31108555985e596438209023d1d99b85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 06:52:12 +0200 Subject: [PATCH 07/48] Add some tests for paramfile writing --- tests/optimize/test_hyperopttools.py | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py index 94216f2f7..7eb18e432 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopttools.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Dict, List import pytest +import rapidjson from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_tools import HyperoptTools @@ -144,3 +145,57 @@ def test___pprint(): "buy_what": "asdf", "buy_notoptimied": 55, # value loaded from strategy }""" + + +def test_get_strategy_filename(default_conf): + + x = HyperoptTools.get_strategy_filename(default_conf, 'DefaultStrategy') + assert isinstance(x, Path) + assert x == Path(__file__).parents[1] / 'strategy/strats/default_strategy.py' + + x = HyperoptTools.get_strategy_filename(default_conf, 'NonExistingStrategy') + assert x is None + + +def test_export_params(tmpdir): + + filename = Path(tmpdir) / "DefaultStrategy.json" + assert not filename.is_file() + params = { + "params_details": { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + "roi": { + "0": 0.528, + "346": 0.08499999999999999, + "507": 0.049, + "1595": 0 + } + }, + "params_not_optimized": { + "stoploss": -0.05, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + } + + } + HyperoptTools.export_params(params, "DefaultStrategy", filename) + + assert filename.is_file() + + content = rapidjson.load(filename.open('r')) + assert content['strategy_name'] == 'DefaultStrategy' + assert 'params' in content + assert "buy" in content["params"] + assert "sell" in content["params"] + assert "roi" in content["params"] + assert "stoploss" in content["params"] + assert "trailing" in content["params"] From aa5181ca81d4268490f6b085fab7696a6e367f9b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 20:14:31 +0200 Subject: [PATCH 08/48] Properly export non-optimized parameters --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c23884bcd..ea75028b4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -208,9 +208,9 @@ class Hyperopt: result: Dict[str, Any] = {} strategy = self.backtesting.strategy if not HyperoptTools.has_space(self.config, 'roi'): - result['roi'] = strategy.minimal_roi + result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()} if not HyperoptTools.has_space(self.config, 'stoploss'): - result['stoploss'] = strategy.stoploss + result['stoploss'] = {'stoploss': strategy.stoploss} if not HyperoptTools.has_space(self.config, 'trailing'): result['trailing'] = { 'trailing_stop': strategy.trailing_stop, From 8b7010fc9a98aa20f38bd71c6861b19d351c61f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 20:15:20 +0200 Subject: [PATCH 09/48] Update pprint name --- freqtrade/optimize/hyperopt_tools.py | 4 ++-- tests/optimize/test_hyperopttools.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 0d17a5d13..7b14440fc 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -186,7 +186,7 @@ class HyperoptTools(): # Buy / sell parameters no_params = HyperoptTools._space_params(non_optimized, space, 5) - result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}" + result += f"{space}_params = {HyperoptTools.__pprint_dict(space_params, no_params)}" result = result.replace("\n", "\n ") print(result) @@ -200,7 +200,7 @@ class HyperoptTools(): return {} @staticmethod - def _pprint(params, non_optimized, indent: int = 4): + def __pprint_dict(params, non_optimized, indent: int = 4): """ Pretty-print hyperopt results (based on 2 dicts - with add. comment) """ diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py index 7eb18e432..69c7073c0 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopttools.py @@ -133,11 +133,11 @@ def test_show_epoch_details(capsys): assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) -def test___pprint(): +def test___pprint_dict(): params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} non_params = {'buy_notoptimied': 55} - x = HyperoptTools._pprint(params, non_params) + x = HyperoptTools.__pprint_dict(params, non_params) assert x == """{ "buy_std": 1.2, "buy_rsi": 31, @@ -171,7 +171,7 @@ def test_export_params(tmpdir): }, "roi": { "0": 0.528, - "346": 0.08499999999999999, + "346": 0.08499, "507": 0.049, "1595": 0 } From a7e9e362b7c2e44f8f5f76281ca7775d5cef0f15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 20:15:37 +0200 Subject: [PATCH 10/48] Simplify printing logic for non-optimized parameters --- freqtrade/optimize/hyperopt_tools.py | 36 ++++++++-------- tests/optimize/test_hyperopttools.py | 61 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 7b14440fc..a99859fd5 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -12,11 +12,13 @@ from pandas import isna, json_normalize from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException -from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict +from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 logger = logging.getLogger(__name__) +NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" + class HyperoptTools(): @@ -158,33 +160,31 @@ class HyperoptTools(): def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None: if space in params or space in non_optimized: space_params = HyperoptTools._space_params(params, space, 5) + no_params = HyperoptTools._space_params(non_optimized, space, 5) + if not space_params and not no_params: + # No parameters - don't print + return + if not space_params: + # Not optimized parameters - append string + non_optimized = NON_OPT_PARAM_APPENDIX + result = f"\n# {header}\n" if space == "stoploss": - opt = True - if not space_params: - space_params = HyperoptTools._space_params(params, space, 5) - opt = False - result += (f"stoploss = {space_params.get('stoploss')}" - f"{' # value loaded from strategy' if not opt else ''}") + stoploss = safe_value_fallback2(space_params, no_params, space, space) + result += (f"stoploss = {stoploss}{non_optimized}") elif space == "roi": + result = result[:-1] + f'{non_optimized}\n' minimal_roi_result = rapidjson.dumps({ - str(k): v for k, v in space_params.items() + str(k): v for k, v in (space_params or no_params).items() }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" elif space == "trailing": - opt = True - if not space_params: - # Not optimized ... - space_params = HyperoptTools._space_params(non_optimized, space, 5) - opt = False - - for k, v in space_params.items(): - result += f"{k} = {v}{' # value loaded from strategy' if not opt else ''}\n" + for k, v in (space_params or no_params).items(): + result += f"{k} = {v}{non_optimized}\n" else: # Buy / sell parameters - no_params = HyperoptTools._space_params(non_optimized, space, 5) result += f"{space}_params = {HyperoptTools.__pprint_dict(space_params, no_params)}" @@ -212,7 +212,7 @@ class HyperoptTools(): result += " " * indent + f'"{k}": ' result += f'"{param}",' if isinstance(param, str) else f'{param},' if k in non_optimized: - result += " # value loaded from strategy" + result += NON_OPT_PARAM_APPENDIX result += "\n" result += '}' return result diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py index 69c7073c0..42b08c23d 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopttools.py @@ -199,3 +199,64 @@ def test_export_params(tmpdir): assert "roi" in content["params"] assert "stoploss" in content["params"] assert "trailing" in content["params"] + + +def test_params_print(capsys): + + params = { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + } + non_optimized = { + "buy": { + "buy_adx": 44 + }, + "sell": { + "sell_adx": 65 + }, + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.05, + "20": 0.01, + }, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + + } + HyperoptTools._params_pretty_print(params, 'buy', 'No header', non_optimized) + + captured = capsys.readouterr() + assert re.search("# No header", captured.out) + assert re.search('"buy_rsi": 30,\n', captured.out) + assert re.search('"buy_adx": 44, # value loaded.*\n', captured.out) + assert not re.search("sell", captured.out) + + HyperoptTools._params_pretty_print(params, 'sell', 'Sell Header', non_optimized) + captured = capsys.readouterr() + assert re.search("# Sell Header", captured.out) + assert re.search('"sell_rsi": 70,\n', captured.out) + assert re.search('"sell_adx": 65, # value loaded.*\n', captured.out) + + HyperoptTools._params_pretty_print(params, 'roi', 'ROI Table:', non_optimized) + captured = capsys.readouterr() + assert re.search("# ROI Table: # value loaded.*\n", captured.out) + assert re.search('minimal_roi = {\n', captured.out) + assert re.search('"20": 0.01\n', captured.out) + + HyperoptTools._params_pretty_print(params, 'trailing', 'Trailing stop:', non_optimized) + captured = capsys.readouterr() + assert re.search("# Trailing stop:", captured.out) + assert re.search('trailing_stop = False # value loaded.*\n', captured.out) + assert re.search('trailing_stop_positive = 0.05 # value loaded.*\n', captured.out) + assert re.search('trailing_stop_positive_offset = 0.1 # value loaded.*\n', captured.out) + assert re.search('trailing_only_offset_is_reached = True # value loaded.*\n', captured.out) From d4514f5f16fd0ef22cd8732908cbd79858ab2690 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 20:27:46 +0200 Subject: [PATCH 11/48] Introduce File versions to hyperopt result files --- freqtrade/commands/hyperopt_commands.py | 16 +++++++++------- freqtrade/constants.py | 1 + freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/optimize/hyperopt_tools.py | 3 ++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index e5c9241f0..e60fb9d32 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration +from freqtrade.constants import FTHYPT_FILEVERSION from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -133,13 +134,14 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: show_backtest_result(strategy_name, metrics, metrics['stake_currency']) - # Export parameters ... - # TODO: make this optional? otherwise it'll overwrite previous parameters ... - fn = HyperoptTools.get_strategy_filename(config, strategy_name) - if fn: - HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) - else: - logger.warn("Strategy not found, not exporting parameter file.") + if val.get(FTHYPT_FILEVERSION, 1) >= 2: + # Export parameters ... + # TODO: make this optional? otherwise it'll overwrite previous parameters ... + fn = HyperoptTools.get_strategy_filename(config, strategy_name) + if fn: + HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + else: + logger.warn("Strategy not found, not exporting parameter file.") HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 63cf3e870..bdbfbfad6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -40,6 +40,7 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] LAST_BT_RESULT_FN = '.last_result.json' +FTHYPT_FILEVERSION = 'fthypt_fileversion' USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ea75028b4..23f47612b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -20,7 +20,7 @@ from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from pandas import DataFrame -from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN +from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange from freqtrade.misc import deep_merge_dicts, file_dump_json, plural @@ -167,7 +167,7 @@ class Hyperopt: if isinstance(x, np.integer): return int(x) return str(x) - + epoch[FTHYPT_FILEVERSION] = 2 with self.results_file.open('a') as f: rapidjson.dump(epoch, f, default=default_parser, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index a99859fd5..8f69fbcff 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -46,7 +46,8 @@ class HyperoptTools(): final_params = deep_merge_dicts(params['params_details'], final_params) final_params = { 'strategy_name': strategy_name, - 'params': final_params + 'params': final_params, + 'ft_stratparam_v': 1, } logger.info(f"Dumping parameters to {filename}") rapidjson.dump(final_params, filename.open('w'), indent=2) From 8ca0076332f45792326e7cfd27cacbbbf9bca5e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Jun 2021 20:33:35 +0200 Subject: [PATCH 12/48] Fix small typos --- freqtrade/optimize/hyperopt_tools.py | 13 +++++++------ tests/optimize/test_hyperopttools.py | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 8f69fbcff..5a9049192 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -162,32 +162,33 @@ class HyperoptTools(): if space in params or space in non_optimized: space_params = HyperoptTools._space_params(params, space, 5) no_params = HyperoptTools._space_params(non_optimized, space, 5) + appendix = '' if not space_params and not no_params: # No parameters - don't print return if not space_params: # Not optimized parameters - append string - non_optimized = NON_OPT_PARAM_APPENDIX + appendix = NON_OPT_PARAM_APPENDIX result = f"\n# {header}\n" if space == "stoploss": stoploss = safe_value_fallback2(space_params, no_params, space, space) - result += (f"stoploss = {stoploss}{non_optimized}") + result += (f"stoploss = {stoploss}{appendix}") elif space == "roi": - result = result[:-1] + f'{non_optimized}\n' + result = result[:-1] + f'{appendix}\n' minimal_roi_result = rapidjson.dumps({ str(k): v for k, v in (space_params or no_params).items() }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" elif space == "trailing": for k, v in (space_params or no_params).items(): - result += f"{k} = {v}{non_optimized}\n" + result += f"{k} = {v}{appendix}\n" else: # Buy / sell parameters - result += f"{space}_params = {HyperoptTools.__pprint_dict(space_params, no_params)}" + result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}" result = result.replace("\n", "\n ") print(result) @@ -201,7 +202,7 @@ class HyperoptTools(): return {} @staticmethod - def __pprint_dict(params, non_optimized, indent: int = 4): + def _pprint_dict(params, non_optimized, indent: int = 4): """ Pretty-print hyperopt results (based on 2 dicts - with add. comment) """ diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py index 42b08c23d..54e968143 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopttools.py @@ -133,11 +133,11 @@ def test_show_epoch_details(capsys): assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) -def test___pprint_dict(): +def test__pprint_dict(): params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} non_params = {'buy_notoptimied': 55} - x = HyperoptTools.__pprint_dict(params, non_params) + x = HyperoptTools._pprint_dict(params, non_params) assert x == """{ "buy_std": 1.2, "buy_rsi": 31, From a2ccc1526e30f2e1e68032464173c742c152ca90 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Jun 2021 07:07:34 +0200 Subject: [PATCH 13/48] Load parameters from file --- freqtrade/resolvers/strategy_resolver.py | 15 +++++++++++++++ freqtrade/strategy/hyper.py | 3 ++- freqtrade/strategy/interface.py | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index ccd7cea69..1239b78b3 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -53,6 +53,21 @@ class StrategyResolver(IResolver): ) strategy.timeframe = strategy.ticker_interval + if strategy._ft_params_from_file: + # Set parameters from Hyperopt results file + params = strategy._ft_params_from_file + strategy.minimal_roi = params.get('roi', strategy.minimal_roi) + + strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss) + trailing = params.get('trailing', {}) + strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop) + strategy.trailing_stop_positive = trailing.get('trailing_stop_positive', + strategy.trailing_stop_positive) + strategy.trailing_stop_positive_offset = trailing.get( + 'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset) + strategy.trailing_only_offset_is_reached = trailing.get( + 'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached) + # Set attributes # Check if we need to override configuration # (Attribute name, default, subkey) diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 0ced4dfb1..6f96224ee 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -309,6 +309,7 @@ class HyperStrategyMixin(object): """ params = self.load_params_from_file() params = params.get('params', {}) + self._ft_params_from_file = params buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', None)) sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', None)) @@ -324,7 +325,7 @@ class HyperStrategyMixin(object): if filename.is_file(): logger.info(f"Loading parameters from file {filename}") params = json_load(filename.open('r')) - if params.get('strategy_name') != self.get_strategy_name(): + if params.get('strategy_name') != self.__class__.__name__: raise OperationalException('Invalid parameter file provided') return params logger.info("Found no parameter file.") diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7aa7e57d9..26bcb0369 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -62,6 +62,7 @@ class IStrategy(ABC, HyperStrategyMixin): _populate_fun_len: int = 0 _buy_fun_len: int = 0 _sell_fun_len: int = 0 + _ft_params_from_file: Dict = {} # associated minimal roi minimal_roi: Dict From 62cdbdc26a339f84a6902ca3f732ccf3254441fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Jun 2021 20:22:30 +0200 Subject: [PATCH 14/48] Automatically export hyperopt parameters --- freqtrade/commands/arguments.py | 5 +++-- freqtrade/commands/cli_options.py | 5 +++++ freqtrade/commands/hyperopt_commands.py | 10 +--------- freqtrade/configuration/configuration.py | 2 ++ freqtrade/optimize/hyperopt.py | 5 +++++ freqtrade/optimize/hyperopt_tools.py | 12 +++++++++++- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 7f4f7edd6..ba37237f6 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -29,7 +29,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_loss"] + "hyperopt_loss", "disableparamexport"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] @@ -85,7 +85,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperoptexportfilename", "export_csv"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", - "print_json", "hyperoptexportfilename", "hyperopt_show_no_header"] + "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", + "disableparamexport"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b226415e7..f56a2bf18 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -178,6 +178,11 @@ AVAILABLE_CLI_OPTIONS = { 'Example: `--export-filename=user_data/backtest_results/backtest_today.json`', metavar='PATH', ), + "disableparamexport": Arg( + '--disable-param-export', + help="Disable automatic hyperopt parameter export.", + action='store_true', + ), "fee": Arg( '--fee', help='Specify fee ratio. Will be applied twice (on trade entry and exit).', diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index e60fb9d32..5a2727795 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -5,7 +5,6 @@ from typing import Any, Dict, List from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration -from freqtrade.constants import FTHYPT_FILEVERSION from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -134,14 +133,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: show_backtest_result(strategy_name, metrics, metrics['stake_currency']) - if val.get(FTHYPT_FILEVERSION, 1) >= 2: - # Export parameters ... - # TODO: make this optional? otherwise it'll overwrite previous parameters ... - fn = HyperoptTools.get_strategy_filename(config, strategy_name) - if fn: - HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) - else: - logger.warn("Strategy not found, not exporting parameter file.") + HyperoptTools.try_export_params(config, strategy_name, val) HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index d2cc68c44..1d2e3f802 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -260,6 +260,8 @@ class Configuration: self._args_to_config(config, argname='export', logstring='Parameter --export detected: {} ...') + self._args_to_config(config, argname='disableparamexport', + logstring='Parameter --disableparamexport detected: {} ...') # Edge section: if 'stoploss_range' in self.args and self.args["stoploss_range"]: txt_range = eval(self.args["stoploss_range"]) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 23f47612b..435273619 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -489,6 +489,11 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) + HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) else: diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 5a9049192..0f8ccbca4 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -10,7 +10,7 @@ import tabulate from colorama import Fore, Style from pandas import isna, json_normalize -from freqtrade.constants import USERPATH_STRATEGIES +from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 @@ -52,6 +52,16 @@ class HyperoptTools(): logger.info(f"Dumping parameters to {filename}") rapidjson.dump(final_params, filename.open('w'), indent=2) + @staticmethod + def try_export_params(config: Dict[str, Any], strategy_name: str, val: Dict): + if val.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): + # Export parameters ... + fn = HyperoptTools.get_strategy_filename(config, strategy_name) + if fn: + HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + else: + logger.warn("Strategy not found, not exporting parameter file.") + @staticmethod def has_space(config: Dict[str, Any], space: str) -> bool: """ From 55f032b18e70693c86e412101e9b5e6d6be872f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Jun 2021 20:38:14 +0200 Subject: [PATCH 15/48] Catch trying to read faulty parameter file --- freqtrade/constants.py | 1 + freqtrade/strategy/hyper.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index bdbfbfad6..f4c32387b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -313,6 +313,7 @@ CONF_SCHEMA = { }, 'db_url': {'type': 'string'}, 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, + 'disableparamexport': {'type': 'boolean'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 6f96224ee..881d592d9 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -324,10 +324,14 @@ class HyperStrategyMixin(object): if filename.is_file(): logger.info(f"Loading parameters from file {filename}") - params = json_load(filename.open('r')) - if params.get('strategy_name') != self.__class__.__name__: - raise OperationalException('Invalid parameter file provided') - return params + try: + params = json_load(filename.open('r')) + if params.get('strategy_name') != self.__class__.__name__: + raise OperationalException('Invalid parameter file provided') + return params + except ValueError: + logger.warning("Invalid parameter file.") + return {} logger.info("Found no parameter file.") return {} From 84703080b812da4cbb2fa5bd5ac0ee5c65710f29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Jun 2021 20:39:07 +0200 Subject: [PATCH 16/48] Extract hyperopt_defaults_serializer to hyperopt_tools --- freqtrade/optimize/hyperopt.py | 8 ++------ freqtrade/optimize/hyperopt_tools.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 435273619..b8745a644 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -29,7 +29,7 @@ from freqtrade.optimize.backtesting import Backtesting from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 -from freqtrade.optimize.hyperopt_tools import HyperoptTools +from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_parser from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver @@ -163,13 +163,9 @@ class Hyperopt: While not a valid json object - this allows appending easily. :param epoch: result dictionary for this epoch. """ - def default_parser(x): - if isinstance(x, np.integer): - return int(x) - return str(x) epoch[FTHYPT_FILEVERSION] = 2 with self.results_file.open('a') as f: - rapidjson.dump(epoch, f, default=default_parser, + rapidjson.dump(epoch, f, default=hyperopt_parser, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) f.write("\n") diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 0f8ccbca4..7558232f1 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -2,9 +2,11 @@ import io import logging from copy import deepcopy +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional +import numpy as np import rapidjson import tabulate from colorama import Fore, Style @@ -20,6 +22,12 @@ logger = logging.getLogger(__name__) NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" +def hyperopt_parser(x): + if isinstance(x, np.integer): + return int(x) + return str(x) + + class HyperoptTools(): @staticmethod @@ -48,9 +56,12 @@ class HyperoptTools(): 'strategy_name': strategy_name, 'params': final_params, 'ft_stratparam_v': 1, + 'export_time': datetime.now(timezone.utc), } logger.info(f"Dumping parameters to {filename}") - rapidjson.dump(final_params, filename.open('w'), indent=2) + rapidjson.dump(final_params, filename.open('w'), indent=2, + default=hyperopt_parser, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN + ) @staticmethod def try_export_params(config: Dict[str, Any], strategy_name: str, val: Dict): From ff61b8a2e795de0687ca027c0b36b1efa260a1d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Jun 2021 20:57:16 +0200 Subject: [PATCH 17/48] Disable parameter export from tests --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index a843d9397..87276456f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -324,6 +324,7 @@ def get_default_conf(testdatadir): "verbosity": 3, "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", + "disableparamexport": True, "internals": {}, "export": "none", } From dcf53ac3ff645de6166c43a84e68b9ce5910a5db Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 06:33:40 +0200 Subject: [PATCH 18/48] Add test for try_eport_params --- freqtrade/optimize/hyperopt_tools.py | 6 ++-- tests/optimize/test_hyperopttools.py | 47 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 7558232f1..7a0b00d01 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -64,12 +64,12 @@ class HyperoptTools(): ) @staticmethod - def try_export_params(config: Dict[str, Any], strategy_name: str, val: Dict): - if val.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): + def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict): + if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): # Export parameters ... fn = HyperoptTools.get_strategy_filename(config, strategy_name) if fn: - HyperoptTools.export_params(val, strategy_name, fn.with_suffix('.json')) + HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json')) else: logger.warn("Strategy not found, not exporting parameter file.") diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopttools.py index 54e968143..6beb2788a 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopttools.py @@ -6,6 +6,7 @@ from typing import Dict, List import pytest import rapidjson +from freqtrade.constants import FTHYPT_FILEVERSION from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_tools import HyperoptTools from tests.conftest import log_has, log_has_re @@ -201,6 +202,52 @@ def test_export_params(tmpdir): assert "trailing" in content["params"] +def test_try_export_params(default_conf, tmpdir, caplog, mocker): + default_conf['disableparamexport'] = False + export_mock = mocker.patch("freqtrade.optimize.hyperopt_tools.HyperoptTools.export_params") + + filename = Path(tmpdir) / "DefaultStrategy.json" + assert not filename.is_file() + params = { + "params_details": { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + "roi": { + "0": 0.528, + "346": 0.08499, + "507": 0.049, + "1595": 0 + } + }, + "params_not_optimized": { + "stoploss": -0.05, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + }, + FTHYPT_FILEVERSION: 2, + + } + HyperoptTools.try_export_params(default_conf, "DefaultStrategy22", params) + + assert log_has("Strategy not found, not exporting parameter file.", caplog) + assert export_mock.call_count == 0 + caplog.clear() + + HyperoptTools.try_export_params(default_conf, "DefaultStrategy", params) + + assert export_mock.call_count == 1 + assert export_mock.call_args_list[0][0][1] == 'DefaultStrategy' + assert export_mock.call_args_list[0][0][2].name == 'default_strategy.json' + + def test_params_print(capsys): params = { From 645da51b5fa01003ad1292fb2f0910ada0347e2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 06:43:49 +0200 Subject: [PATCH 19/48] Add test for parameter loading --- freqtrade/optimize/hyperopt.py | 1 - freqtrade/strategy/hyper.py | 4 +-- tests/optimize/test_hyperopt.py | 2 -- tests/strategy/test_interface.py | 48 ++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8745a644..b22aa58c5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -12,7 +12,6 @@ from math import ceil from pathlib import Path from typing import Any, Dict, List, Optional -import numpy as np import progressbar import rapidjson from colorama import Fore, Style diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 881d592d9..a31a3b39f 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -327,10 +327,10 @@ class HyperStrategyMixin(object): try: params = json_load(filename.open('r')) if params.get('strategy_name') != self.__class__.__name__: - raise OperationalException('Invalid parameter file provided') + raise OperationalException('Invalid parameter file provided.') return params except ValueError: - logger.warning("Invalid parameter file.") + logger.warning("Invalid parameter file format.") return {} logger.info("Found no parameter file.") diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 91d9f5496..f0a2342c5 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1020,5 +1020,3 @@ def test_SKDecimal(): assert space.transform([2.0]) == [200] assert space.transform([1.0]) == [100] assert space.transform([1.5, 1.6]) == [150, 160] - - diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 04d12a51f..714e28929 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock import arrow @@ -692,3 +693,50 @@ def test_auto_hyperopt_interface(default_conf): with pytest.raises(OperationalException, match=r"Inconclusive parameter.*"): [x for x in strategy.detect_parameters('sell')] + + +def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): + default_conf.update({'strategy': 'HyperoptableStrategy'}) + del default_conf['stoploss'] + del default_conf['minimal_roi'] + mocker.patch.object(Path, 'is_file', MagicMock(return_value=True)) + mocker.patch.object(Path, 'open') + expected_result = { + "strategy_name": "HyperoptableStrategy", + "params": { + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.2, + "1200": 0.01 + } + } + } + mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + PairLocks.timeframe = default_conf['timeframe'] + strategy = StrategyResolver.load_strategy(default_conf) + assert strategy.stoploss == -0.05 + assert strategy.minimal_roi == {0: 0.2, 1200: 0.01} + + expected_result = { + "strategy_name": "HyperoptableStrategy_No", + "params": { + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.2, + "1200": 0.01 + } + } + } + + mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + with pytest.raises(OperationalException, match="Invalid parameter file provided."): + StrategyResolver.load_strategy(default_conf) + + mocker.patch('freqtrade.strategy.hyper.json_load', MagicMock(side_effect=ValueError())) + + StrategyResolver.load_strategy(default_conf) + assert log_has("Invalid parameter file format.", caplog) From 0809225a0ac07f4f056053791c1c13efdd571a2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 07:05:20 +0200 Subject: [PATCH 20/48] Update documentation to mention parameter strategy files --- docs/hyperopt.md | 15 +++++++++++++-- docs/utils.md | 5 ++++- freqtrade/optimize/hyperopt.py | 12 ++++++++---- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a117ac1ce..5dee63256 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -51,7 +51,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] - [--hyperopt-loss NAME] + [--hyperopt-loss NAME] [--disable-param-export] optional arguments: -h, --help show this help message and exit @@ -118,6 +118,8 @@ optional arguments: ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, SortinoHyperOptLoss, SortinoHyperOptLossDaily + --disable-param-export + Disable automatic hyperopt parameter export. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -509,7 +511,13 @@ You should understand this result like: * You should not use ADX because `'buy_adx_enabled': False`. * You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`) -Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed. +### Automatic parameter application to the strategy + +When using Hyperoptable parameters, the result of your hyperopt-run will be written to a json file next to your strategy (so for `MyAwesomeStrategy.py`, the file would be `MyAwesomeStrategy.json`). +This file is also updated when using the `hyperopt-show` sub-command, unless `--disable-param-export` is provided to either of the 2 commands. + + +Your strategy class can also contain these results explicitly. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed. Transferring your whole hyperopt result to your strategy would then look like: @@ -525,6 +533,9 @@ class MyAwesomeStrategy(IStrategy): } ``` +!!! Note: + Parameter-files will overwrite parameters within the strategy. + ### Understand Hyperopt ROI results If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table: diff --git a/docs/utils.md b/docs/utils.md index 8ef12e1c9..524fefc21 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -702,7 +702,8 @@ You can show the details of any hyperoptimization epoch previously evaluated by usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] [--profitable] [-n INT] [--print-json] - [--hyperopt-filename PATH] [--no-header] + [--hyperopt-filename FILENAME] [--no-header] + [--disable-param-export] optional arguments: -h, --help show this help message and exit @@ -714,6 +715,8 @@ optional arguments: Hyperopt result filename.Example: `--hyperopt- filename=hyperopt_results_2020-09-27_16-20-48.pickle` --no-header Do not print epoch details header. + --disable-param-export + Disable automatic hyperopt parameter export. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b22aa58c5..1f50d9a16 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -77,8 +77,11 @@ class Hyperopt: if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) + self.auto_hyperopt = True else: self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) + self.auto_hyperopt = False + self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy @@ -484,10 +487,11 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - HyperoptTools.try_export_params( - self.config, - self.backtesting.strategy.get_strategy_name(), - self.current_best_epoch) + if self.auto_hyperopt: + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) From 15e36a20e1948e891a1c7956e5db3a6e09b1c26b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 19:48:34 +0200 Subject: [PATCH 21/48] Improve naming of default hyperopt serializer --- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/optimize/hyperopt_tools.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1f50d9a16..73a04a8cb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -28,7 +28,7 @@ from freqtrade.optimize.backtesting import Backtesting from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 -from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_parser +from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver @@ -167,7 +167,7 @@ class Hyperopt: """ epoch[FTHYPT_FILEVERSION] = 2 with self.results_file.open('a') as f: - rapidjson.dump(epoch, f, default=hyperopt_parser, + rapidjson.dump(epoch, f, default=hyperopt_serializer, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) f.write("\n") diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 7a0b00d01..006bc4ce0 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" -def hyperopt_parser(x): +def hyperopt_serializer(x): if isinstance(x, np.integer): return int(x) return str(x) @@ -60,7 +60,8 @@ class HyperoptTools(): } logger.info(f"Dumping parameters to {filename}") rapidjson.dump(final_params, filename.open('w'), indent=2, - default=hyperopt_parser, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN + default=hyperopt_serializer, + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN ) @staticmethod From 60b7f6edff03bb90088c8278dad42dfae4c197d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 19:53:36 +0200 Subject: [PATCH 22/48] Improve documentation --- docs/hyperopt.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 5dee63256..bfa198f61 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -533,8 +533,9 @@ class MyAwesomeStrategy(IStrategy): } ``` -!!! Note: - Parameter-files will overwrite parameters within the strategy. +!!! Note + Values in the configuration file will overwrite Parameter-file level parameters - and both will overwrite parameters within the strategy. + The prevalence is therefore: config > parameter file > strategy ### Understand Hyperopt ROI results From e034f11dcced3a9b1e5a6bfd674451e5657836a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Jun 2021 20:21:33 +0200 Subject: [PATCH 23/48] Improve test for hyperopt_show --- tests/commands/test_commands.py | 1 + tests/conftest.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 47f298ad7..dcceb3ea1 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1168,6 +1168,7 @@ def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', MagicMock(return_value=saved_hyperopt_results) ) + mocker.patch('freqtrade.commands.hyperopt_commands.show_backtest_result') args = [ "hyperopt-show", diff --git a/tests/conftest.py b/tests/conftest.py index 87276456f..c21458e66 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1954,12 +1954,13 @@ def saved_hyperopt_results(): 'params_dict': { 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 - 'results_metrics': {'total_trades': 2, 'wins': 0, 'draws': 0, 'losses': 2, 'profit_mean': -0.01254995, 'profit_median': -0.012222, 'profit_total': -0.00125625, 'profit_total_abs': -2.50999, 'holding_avg': timedelta(minutes=3930.0)}, # noqa: E501 + 'results_metrics': {'total_trades': 2, 'wins': 0, 'draws': 0, 'losses': 2, 'profit_mean': -0.01254995, 'profit_median': -0.012222, 'profit_total': -0.00125625, 'profit_total_abs': -2.50999, 'holding_avg': timedelta(minutes=3930.0), 'stake_currency': 'BTC', 'strategy_name': 'SampleStrategy'}, # noqa: E501 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 'total_profit': -0.00125625, 'current_epoch': 1, 'is_initial_point': True, - 'is_best': True + 'is_best': True, + }, { 'loss': 20.0, 'params_dict': { From b25ad68c4482d7e4588986e179d14cd318f4ef16 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Jul 2021 20:52:25 +0200 Subject: [PATCH 24/48] Fix np.bool_ not outputting correctly --- freqtrade/optimize/hyperopt_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 006bc4ce0..90976d34e 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -25,6 +25,9 @@ NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" def hyperopt_serializer(x): if isinstance(x, np.integer): return int(x) + if isinstance(x, np.bool_): + return bool(x) + return str(x) From 3503fdb4ec31be99f433fdce039543e0911964d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jul 2021 08:38:55 +0200 Subject: [PATCH 25/48] Improve tests for newly added methods --- tests/optimize/test_hyperopt.py | 12 ++++++++++++ ...{test_hyperopttools.py => test_hyperopt_tools.py} | 10 +++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) rename tests/optimize/{test_hyperopttools.py => test_hyperopt_tools.py} (97%) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index f0a2342c5..14fea573f 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -307,6 +307,18 @@ def test_roi_table_generation(hyperopt) -> None: assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} +def test_params_no_optimize_details(hyperopt) -> None: + hyperopt.config['spaces'] = ['buy'] + res = hyperopt._get_no_optimize_details() + assert isinstance(res, dict) + assert "trailing" in res + assert res["trailing"]['trailing_stop'] is False + assert "roi" in res + assert res['roi']['0'] == 0.04 + assert "stoploss" in res + assert res['stoploss']['stoploss'] == -0.1 + + def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') diff --git a/tests/optimize/test_hyperopttools.py b/tests/optimize/test_hyperopt_tools.py similarity index 97% rename from tests/optimize/test_hyperopttools.py rename to tests/optimize/test_hyperopt_tools.py index 6beb2788a..72125f1a2 100644 --- a/tests/optimize/test_hyperopttools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -2,13 +2,14 @@ import logging import re from pathlib import Path from typing import Dict, List +import numpy as np import pytest import rapidjson from freqtrade.constants import FTHYPT_FILEVERSION from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_tools import HyperoptTools +from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from tests.conftest import log_has, log_has_re @@ -307,3 +308,10 @@ def test_params_print(capsys): assert re.search('trailing_stop_positive = 0.05 # value loaded.*\n', captured.out) assert re.search('trailing_stop_positive_offset = 0.1 # value loaded.*\n', captured.out) assert re.search('trailing_only_offset_is_reached = True # value loaded.*\n', captured.out) + + +def test_hyperopt_serializer(): + + assert isinstance(hyperopt_serializer(np.int_(5)), int) + assert isinstance(hyperopt_serializer(np.bool_(True)), bool) + assert isinstance(hyperopt_serializer(np.bool_(False)), bool) From 3686efa08a168ded9f801972840255402616a83a Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 3 Jul 2021 10:08:52 +0300 Subject: [PATCH 26/48] Add range property to CategoricalParameter and DecimalParameter, add their tests. At the moment we can keep a single code path when using IntParameter, but we have to make a special hyperopt case for CategoricalParameter/DecimalParameter. Range property solves this. --- docs/hyperopt.md | 3 +++ freqtrade/strategy/hyper.py | 28 ++++++++++++++++++++++++++++ tests/strategy/test_interface.py | 25 ++++++++++++++++++++----- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index a117ac1ce..5ce0d3813 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -403,6 +403,9 @@ While this strategy is most likely too simple to provide consistent profit, it s !!! Note `self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`). +!!! Note + `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space. + ??? Hint "Performance tip" By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 21a806202..36a3b0600 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -205,6 +205,21 @@ class DecimalParameter(NumericParameter): return SKDecimal(low=self.low, high=self.high, decimals=self._decimals, name=name, **self._space_params) + @property + def range(self): + """ + Get each value in this space as list. + Returns a List from low to high (inclusive) in Hyperopt mode. + Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid + calculating 100ds of indicators. + """ + if self.in_space and self.optimize: + low = int(self.low * pow(10, self._decimals)) + high = int(self.high * pow(10, self._decimals)) + 1 + return [round(n * pow(0.1, self._decimals), self._decimals) for n in range(low, high)] + else: + return [self.value] + class CategoricalParameter(BaseParameter): default: Any @@ -239,6 +254,19 @@ class CategoricalParameter(BaseParameter): """ return Categorical(self.opt_range, name=name, **self._space_params) + @property + def range(self): + """ + Get each value in this space as list. + Returns a List of categories in Hyperopt mode. + Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid + calculating 100ds of indicators. + """ + if self.in_space and self.optimize: + return self.opt_range + else: + return [self.value] + class HyperStrategyMixin(object): """ diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 04d12a51f..2beb4465d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -12,6 +12,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.hyper import (BaseParameter, CategoricalParameter, DecimalParameter, @@ -657,17 +658,31 @@ def test_hyperopt_parameters(): assert list(intpar.range) == [0, 1, 2, 3, 4, 5] fltpar = RealParameter(low=0.0, high=5.5, default=1.0, space='buy') + assert fltpar.value == 1 assert isinstance(fltpar.get_space(''), Real) - assert fltpar.value == 1 - fltpar = DecimalParameter(low=0.0, high=5.5, default=1.0004, decimals=3, space='buy') - assert isinstance(fltpar.get_space(''), Integer) - assert fltpar.value == 1 + fltpar = DecimalParameter(low=0.0, high=0.5, default=0.14, decimals=1, space='buy') + assert fltpar.value == 0.1 + assert isinstance(fltpar.get_space(''), SKDecimal) + assert isinstance(fltpar.range, list) + assert len(list(fltpar.range)) == 1 + # Range contains ONLY the default / value. + assert list(fltpar.range) == [fltpar.value] + fltpar.in_space = True + assert len(list(fltpar.range)) == 6 + assert list(fltpar.range) == [0.0, 0.1, 0.2, 0.3, 0.4, 0.5] catpar = CategoricalParameter(['buy_rsi', 'buy_macd', 'buy_none'], default='buy_macd', space='buy') - assert isinstance(catpar.get_space(''), Categorical) assert catpar.value == 'buy_macd' + assert isinstance(catpar.get_space(''), Categorical) + assert isinstance(catpar.range, list) + assert len(list(catpar.range)) == 1 + # Range contains ONLY the default / value. + assert list(catpar.range) == [catpar.value] + catpar.in_space = True + assert len(list(catpar.range)) == 3 + assert list(catpar.range) == ['buy_rsi', 'buy_macd', 'buy_none'] def test_auto_hyperopt_interface(default_conf): From dc8abd77df749c661a9c81a839c34524ed9f0e37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jul 2021 15:45:00 +0200 Subject: [PATCH 27/48] Fix import order --- tests/optimize/test_hyperopt_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py index 72125f1a2..44b4a7a03 100644 --- a/tests/optimize/test_hyperopt_tools.py +++ b/tests/optimize/test_hyperopt_tools.py @@ -2,8 +2,8 @@ import logging import re from pathlib import Path from typing import Dict, List -import numpy as np +import numpy as np import pytest import rapidjson From b722e1235019571288216f3408165ac06f8de028 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 02:44:48 +0700 Subject: [PATCH 28/48] compact low balance currencies --- freqtrade/rpc/telegram.py | 15 ++++++++++++--- tests/conftest.py | 2 +- tests/rpc/test_rpc_telegram.py | 11 ++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d857e46a9..7ed564297 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -598,6 +598,9 @@ class Telegram(RPCHandler): "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) + total_dust_balance = 0 + total_dust_currencies = 0 + curr_output = '' for curr in result['currencies']: if curr['est_stake'] > balance_dust_level: curr_output = ( @@ -607,9 +610,9 @@ class Telegram(RPCHandler): f"\t`Pending: {curr['used']:.8f}`\n" f"\t`Est. {curr['stake']}: " f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n") - else: - curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} " - f"{curr['stake']} amount \n") + elif curr['est_stake'] <= balance_dust_level: + total_dust_balance += curr['est_stake'] + total_dust_currencies += 1 # Handle overflowing message length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: @@ -618,6 +621,12 @@ class Telegram(RPCHandler): else: output += curr_output + if total_dust_balance > 0: + output += ( + f"*{total_dust_currencies} Other Currencies:*\n" + f"\t`Est. {result['stake']}: " + f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") + output += ("\n*Estimated Value*:\n" f"\t`{result['stake']}: {result['total']: .8f}`\n" f"\t`{result['symbol']}: " diff --git a/tests/conftest.py b/tests/conftest.py index a843d9397..4a81175ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1761,7 +1761,7 @@ def rpc_balance(): 'total': 0.1, 'free': 0.01, 'used': 0.0 - }, + }, 'EUR': { 'total': 10.0, 'free': 10.0, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 782ae69c6..241805736 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -515,16 +515,21 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick patch_get_signal(freqtradebot, (True, False)) telegram._balance(update=update, context=MagicMock()) + print('msg_mock.call_args_list') + print(msg_mock.call_args_list) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '*BTC:*' in result assert '*ETH:*' not in result - assert '*USDT:*' in result - assert '*EUR:*' in result + assert '*USDT:*' not in result + assert '*EUR:*' not in result + assert '*LTC:*' in result + assert '*XRP:*' not in result assert 'Balance:' in result assert 'Est. BTC:' in result assert 'BTC: 12.00000000' in result - assert '*XRP:* not showing <0.0001 BTC amount' in result + assert "*3 Other Currencies:*" in result + assert 'BTC: 0.00000309' in result def test_balance_handle_empty_response(default_conf, update, mocker) -> None: From dbdd7f38a88b5857dd276456951c168796b5b286 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 02:56:05 +0700 Subject: [PATCH 29/48] add plural --- freqtrade/rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7ed564297..389ef84a4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -11,7 +11,6 @@ from html import escape from itertools import chain from math import isnan from typing import Any, Callable, Dict, List, Optional, Union - import arrow from tabulate import tabulate from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, @@ -24,7 +23,7 @@ from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException -from freqtrade.misc import chunks, round_coin_value +from freqtrade.misc import chunks, round_coin_value, plural from freqtrade.rpc import RPC, RPCException, RPCHandler @@ -623,7 +622,8 @@ class Telegram(RPCHandler): if total_dust_balance > 0: output += ( - f"*{total_dust_currencies} Other Currencies:*\n" + f"*{total_dust_currencies} Other " + f"{plural(total_dust_currencies, 'Currency', 'Currencies')}:*\n" f"\t`Est. {result['stake']}: " f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") From 7efa228d73e538165ffacce578a3d649725aab01 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 03:08:29 +0700 Subject: [PATCH 30/48] add dust balance --- freqtrade/rpc/telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 389ef84a4..0bacf0504 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -623,7 +623,8 @@ class Telegram(RPCHandler): if total_dust_balance > 0: output += ( f"*{total_dust_currencies} Other " - f"{plural(total_dust_currencies, 'Currency', 'Currencies')}:*\n" + f"{plural(total_dust_currencies, 'Currency', 'Currencies')} " + f"(< {balance_dust_level} {result['stake']}):*\n" f"\t`Est. {result['stake']}: " f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") From a4096318e0f0548190bec67686a10a73de9d08e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jul 2021 10:15:19 +0200 Subject: [PATCH 31/48] Provide full backtest-statistics to Hyperopt loss functions closes #5223 --- docs/advanced-hyperopt.md | 4 +++- freqtrade/optimize/hyperopt.py | 3 ++- freqtrade/optimize/hyperopt_loss_interface.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 35fd3de4a..5e71df67c 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -32,6 +32,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, config: Dict, processed: Dict[str, DataFrame], + backtest_stats: Dict[str, Any], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results @@ -53,7 +54,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): Currently, the arguments are: -* `results`: DataFrame containing the result +* `results`: DataFrame containing the resulting trades. The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): `pair, profit_ratio, profit_abs, open_date, open_rate, fee_open, close_date, close_rate, fee_close, amount, trade_duration, is_open, sell_reason, stake_amount, min_rate, max_rate, stop_loss_ratio, stop_loss_abs` * `trade_count`: Amount of trades (identical to `len(results)`) @@ -61,6 +62,7 @@ Currently, the arguments are: * `min_date`: End date of the timerange used * `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space). * `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting. +* `backtest_stats`: Backtesting statistics using the same format as the backtesting file "strategy" substructure. Available fields can be seen in `generate_strategy_stats()` in `optimize_reports.py`. This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c2b2b93cb..a74f0ae3b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -324,7 +324,8 @@ class Hyperopt: loss = self.calculate_loss(results=backtesting_results['results'], trade_count=trade_count, min_date=min_date, max_date=max_date, - config=self.config, processed=processed) + config=self.config, processed=processed, + backtest_stats=strat_stats) return { 'loss': loss, 'params_dict': params_dict, diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index b5aa588b2..ac8239b75 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt from abc import ABC, abstractmethod from datetime import datetime -from typing import Dict +from typing import Any, Dict from pandas import DataFrame @@ -22,6 +22,7 @@ class IHyperOptLoss(ABC): def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, config: Dict, processed: Dict[str, DataFrame], + backtest_stats: Dict[str, Any], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results From 77293b1f1e3ad86e2afdbdf4fcb3797fd1a7ef79 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jul 2021 10:50:10 +0200 Subject: [PATCH 32/48] Remove Zero duration Trades after the recent backtesting fixes, this metric no longer makes sense, as it can't really be 0 any longer. --- docs/backtesting.md | 3 --- freqtrade/optimize/optimize_reports.py | 12 ------------ 2 files changed, 15 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 4899b1dad..89980c670 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -302,7 +302,6 @@ A backtesting result will look like that: | Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | -| Zero Duration Trades | 4.6% (20) | | Rejected Buy signals | 3089 | | | | | Min balance | 0.00945123 BTC | @@ -390,7 +389,6 @@ It contains some useful key metrics about performance of your strategy on backte | Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | -| Zero Duration Trades | 4.6% (20) | | Rejected Buy signals | 3089 | | | | | Min balance | 0.00945123 BTC | @@ -420,7 +418,6 @@ It contains some useful key metrics about performance of your strategy on backte - `Best day` / `Worst day`: Best and worst day based on daily profit. - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade). - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. -- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results. - `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached. - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. - `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 79208c5e9..2a794859d 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -229,8 +229,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: winning_trades = results.loc[results['profit_ratio'] > 0] draw_trades = results.loc[results['profit_ratio'] == 0] losing_trades = results.loc[results['profit_ratio'] < 0] - zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) & - (results['sell_reason'] == 'trailing_stop_loss')]) holding_avg = (timedelta(minutes=round(results['trade_duration'].mean())) if not results.empty else timedelta()) @@ -249,7 +247,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: 'winner_holding_avg_s': winner_holding_avg.total_seconds(), 'loser_holding_avg': loser_holding_avg, 'loser_holding_avg_s': loser_holding_avg.total_seconds(), - 'zero_duration_trades': zero_duration_trades, } @@ -542,14 +539,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show # command stores these results and newer version of freqtrade must be able to handle old # results with missing new fields. - zero_duration_trades = '--' - - if 'zero_duration_trades' in strat_results: - zero_duration_trades_per = \ - 100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades'] - zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \ - f'({strat_results["zero_duration_trades"]})' - metrics = [ ('Backtesting from', strat_results['backtest_start']), ('Backtesting to', strat_results['backtest_end']), @@ -585,7 +574,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), - ('Zero Duration Trades', zero_duration_trades), ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')), ('', ''), # Empty line to improve readability From 791dfd9ba34e7e58cddf9f679f0944fff0fe08ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jul 2021 14:02:11 +0200 Subject: [PATCH 33/48] Fix some doc typos --- docs/strategy-advanced.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 3436604a9..b06cf3ecb 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -55,7 +55,7 @@ class AwesomeStrategy(IStrategy): dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) # Obtain last available candle. Do not use current_time to look up latest candle, because - # current_time points to curret incomplete candle whose data is not available. + # current_time points to current incomplete candle whose data is not available. last_candle = dataframe.iloc[-1].squeeze() # <...> @@ -83,7 +83,7 @@ It is possible to define custom sell signals, indicating that specified position For example you could implement a 1:2 risk-reward ROI with `custom_sell()`. -Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. +Using custom_sell() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. !!! Note Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. @@ -243,7 +243,7 @@ class AwesomeStrategy(IStrategy): current_rate: float, current_profit: float, **kwargs) -> float: if current_profit < 0.04: - return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss + return -1 # return a value bigger than the initial stoploss to keep using the initial stoploss # After reaching the desired offset, allow the stoploss to trail by half the profit desired_stoploss = current_profit / 2 From 4aa2ae37bd6eb8910153fa8ae6043d93340187b8 Mon Sep 17 00:00:00 2001 From: octaviusgus <86911628+octaviusgus@users.noreply.github.com> Date: Sun, 4 Jul 2021 14:38:17 +0200 Subject: [PATCH 34/48] add daily_profit_list added extra key daily_profit in return of optimize_reports.generate_daily_stats this allows us to analyze and plot a daily profit chart / equity line using snippet below inside jupyter notebook ``` # Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day) from freqtrade.configuration import Configuration from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats import plotly.express as px import pandas as pd # strategy = 'Strat' # config = Configuration.from_files(["user_data/config.json"]) # backtest_dir = config["user_data_dir"] / "backtest_results" stats = load_backtest_stats(backtest_dir) strategy_stats = stats['strategy'][strategy] equity = 0 equity_daily = [] for dp in strategy_stats['daily_profit']: equity_daily.append(equity) equity += float(dp) dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end']) df = pd.DataFrame({'dates':dates,'equity_daily':equity_daily}) fig = px.line(df, x="dates", y="equity_daily") fig.show() ``` --- freqtrade/optimize/optimize_reports.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 2a794859d..89cf70437 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -261,6 +261,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: 'winning_days': 0, 'draw_days': 0, 'losing_days': 0, + 'daily_profit_list': [], } daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum() daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10) @@ -271,6 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: winning_days = sum(daily_profit > 0) draw_days = sum(daily_profit == 0) losing_days = sum(daily_profit < 0) + daily_profit_list = daily_profit.tolist() return { 'backtest_best_day': best_rel, @@ -280,6 +282,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: 'winning_days': winning_days, 'draw_days': draw_days, 'losing_days': losing_days, + 'daily_profit': daily_profit_list, } From 558bcc79592a459f45e1f8d4a44d526202d259e1 Mon Sep 17 00:00:00 2001 From: octaviusgus Date: Sun, 4 Jul 2021 15:56:55 +0200 Subject: [PATCH 35/48] Jupyter notebook snippet: Plotting daily profit / equity line --- .../templates/strategy_analysis_example.ipynb | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 0bc593e2d..9af24ae2d 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -188,6 +188,47 @@ "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plotting daily profit / equity line" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)\n", + "\n", + "from freqtrade.configuration import Configuration\n", + "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n", + "import plotly.express as px\n", + "import pandas as pd\n", + "\n", + "# strategy = 'SampleStrategy'\n", + "# config = Configuration.from_files([\"user_data/config.json\"])\n", + "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n", + "\n", + "stats = load_backtest_stats(backtest_dir)\n", + "strategy_stats = stats['strategy'][strategy]\n", + "\n", + "equity = 0\n", + "equity_daily = []\n", + "for dp in strategy_stats['daily_profit']:\n", + " equity_daily.append(equity)\n", + " equity += float(dp)\n", + "\n", + "dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end'])\n", + "\n", + "df = pd.DataFrame({'dates':dates,'equity_daily':equity_daily})\n", + "\n", + "fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n", + "fig.show()\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -329,7 +370,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.8.5" }, "mimetype": "text/x-python", "name": "python", From 9e548657e03d279a143ad304233031d3393faff9 Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 21:08:46 +0700 Subject: [PATCH 36/48] fix testcase --- tests/rpc/test_rpc_telegram.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 241805736..4784f1172 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -515,8 +515,6 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick patch_get_signal(freqtradebot, (True, False)) telegram._balance(update=update, context=MagicMock()) - print('msg_mock.call_args_list') - print(msg_mock.call_args_list) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 assert '*BTC:*' in result @@ -528,7 +526,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick assert 'Balance:' in result assert 'Est. BTC:' in result assert 'BTC: 12.00000000' in result - assert "*3 Other Currencies:*" in result + assert "*3 Other Currencies (< 0.0001 BTC):*" in result assert 'BTC: 0.00000309' in result From c3cf71bba8465675aab47c14233664ce89d58ebe Mon Sep 17 00:00:00 2001 From: kevinjulian Date: Sun, 4 Jul 2021 22:04:39 +0700 Subject: [PATCH 37/48] sort import --- freqtrade/rpc/telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0bacf0504..171a53ca1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -11,6 +11,7 @@ from html import escape from itertools import chain from math import isnan from typing import Any, Callable, Dict, List, Optional, Union + import arrow from tabulate import tabulate from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, @@ -23,7 +24,7 @@ from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException -from freqtrade.misc import chunks, round_coin_value, plural +from freqtrade.misc import chunks, plural, round_coin_value from freqtrade.rpc import RPC, RPCException, RPCHandler From c5489d530a8f62d9666866b1501fa66e581ba175 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jul 2021 19:50:44 +0200 Subject: [PATCH 38/48] Reexport File to docs to have this available as documentation too --- docs/strategy_analysis_example.md | 33 +++++++++++++++++++ .../templates/strategy_analysis_example.ipynb | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 4c938500c..27c620c3d 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -130,6 +130,39 @@ trades = load_backtest_data(backtest_dir) trades.groupby("pair")["sell_reason"].value_counts() ``` +## Plotting daily profit / equity line + + +```python +# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day) + +from freqtrade.configuration import Configuration +from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats +import plotly.express as px +import pandas as pd + +# strategy = 'SampleStrategy' +# config = Configuration.from_files(["user_data/config.json"]) +# backtest_dir = config["user_data_dir"] / "backtest_results" + +stats = load_backtest_stats(backtest_dir) +strategy_stats = stats['strategy'][strategy] + +equity = 0 +equity_daily = [] +for dp in strategy_stats['daily_profit']: + equity_daily.append(equity) + equity += float(dp) + +dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end']) + +df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily}) + +fig = px.line(df, x="dates", y="equity_daily") +fig.show() + +``` + ### Load live trading results into a pandas dataframe In case you did already some trading and want to analyze your performance diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 9af24ae2d..f3b0d8d03 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -197,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -223,7 +223,7 @@ "\n", "dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end'])\n", "\n", - "df = pd.DataFrame({'dates':dates,'equity_daily':equity_daily})\n", + "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n", "\n", "fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n", "fig.show()\n" From 7ac55e5415f94d7db285c20d093a902b039d05de Mon Sep 17 00:00:00 2001 From: nightshift2k Date: Sun, 4 Jul 2021 21:08:42 +0200 Subject: [PATCH 39/48] AgeFilter, RangeStabilityFilter, VolatilityFilter changed `float_timestamp` to `int_timestamp` --- freqtrade/plugins/pairlist/AgeFilter.py | 4 ++-- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- freqtrade/plugins/pairlist/rangestabilityfilter.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8f623b062..09d8588c1 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -64,7 +64,7 @@ class AgeFilter(IPairList): since_ms = int(arrow.utcnow() .floor('day') .shift(days=-self._min_days_listed - 1) - .float_timestamp) * 1000 + .int_timestamp) * 1000 candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): @@ -89,7 +89,7 @@ class AgeFilter(IPairList): if len(daily_candles) >= self._min_days_listed: # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 + self._symbolsChecked[pair] = int(arrow.utcnow().int_timestamp) * 1000 return True else: self.log_once(f"Removed {pair} from whitelist, because age " diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 5ae8e3e9f..a50bf55b9 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -72,7 +72,7 @@ class VolatilityFilter(IPairList): since_ms = int(arrow.utcnow() .floor('day') .shift(days=-self._days - 1) - .float_timestamp) * 1000 + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 8be61166b..6f013c750 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -65,7 +65,7 @@ class RangeStabilityFilter(IPairList): since_ms = int(arrow.utcnow() .floor('day') .shift(days=-self._days - 1) - .float_timestamp) * 1000 + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: From 0d787fde5890f1bcf42284246bd3694da72d85db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 03:01:11 +0000 Subject: [PATCH 40/48] Bump plotly from 5.0.0 to 5.1.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 0563e5df2..e03fd4d66 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.0.0 +plotly==5.1.0 From 2f97846bd87cd5ee3898549303c6282452795b30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 03:01:18 +0000 Subject: [PATCH 41/48] Bump ccxt from 1.52.4 to 1.52.40 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.52.4 to 1.52.40. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.52.4...1.52.40) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a983e4797..d7f918486 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.0 pandas==1.2.5 -ccxt==1.52.4 +ccxt==1.52.40 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 7ae5f47242d1fece46a2422bef8bc292385348da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 03:01:24 +0000 Subject: [PATCH 42/48] Bump sqlalchemy from 1.4.19 to 1.4.20 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.19 to 1.4.20. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a983e4797..7d52cdc2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.52.4 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.19 +SQLAlchemy==1.4.20 python-telegram-bot==13.6 arrow==1.1.1 cachetools==4.2.2 From d1555a10957f32a9c08129ff30995de26fb3cab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 03:01:28 +0000 Subject: [PATCH 43/48] Bump fastapi from 0.65.2 to 0.66.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.65.2 to 0.66.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.65.2...0.66.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a983e4797..3e9131ded 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ python-rapidjson==1.4 sdnotify==0.3.2 # API Server -fastapi==0.65.2 +fastapi==0.66.0 uvicorn==0.14.0 pyjwt==2.1.0 aiofiles==0.7.0 From 0c8afea38271c2519be61179a09160ef6f1806fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 04:30:54 +0000 Subject: [PATCH 44/48] Bump pandas from 1.2.5 to 1.3.0 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.2.5 to 1.3.0. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.2.5...v1.3.0) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1c784608..0003ec031 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.0 -pandas==1.2.5 +pandas==1.3.0 ccxt==1.52.40 # Pin cryptography for now due to rust build errors with piwheels From ac7598ff1450ae94d7569341f045b3aa38e558d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 04:31:00 +0000 Subject: [PATCH 45/48] Bump python-telegram-bot from 13.6 to 13.7 Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 13.6 to 13.7. - [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases) - [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst) - [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v13.6...v13.7) --- updated-dependencies: - dependency-name: python-telegram-bot dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1c784608..b289c13ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.52.40 cryptography==3.4.7 aiohttp==3.7.4.post0 SQLAlchemy==1.4.20 -python-telegram-bot==13.6 +python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 requests==2.25.1 From 5626ca5a06dcbc29cad9a925945351765188ace1 Mon Sep 17 00:00:00 2001 From: nightshift2k Date: Mon, 5 Jul 2021 10:39:22 +0200 Subject: [PATCH 46/48] removed unnecessary casting to int() --- freqtrade/plugins/pairlist/AgeFilter.py | 8 ++++---- freqtrade/plugins/pairlist/VolatilityFilter.py | 8 ++++---- freqtrade/plugins/pairlist/rangestabilityfilter.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 09d8588c1..744b1268d 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -61,10 +61,10 @@ class AgeFilter(IPairList): if not needed_pairs: return pairlist - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._min_days_listed - 1) - .int_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._min_days_listed - 1) + .int_timestamp) * 1000 candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index a50bf55b9..9383e5d06 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -69,10 +69,10 @@ class VolatilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 6f013c750..a6d1820de 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -62,10 +62,10 @@ class RangeStabilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .int_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: From 10998eb0faa8da854b777bdf2b4605caa03dfb7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 5 Jul 2021 19:51:14 +0200 Subject: [PATCH 47/48] Remove further usages of int(int_timestamp) --- freqtrade/data/history/history_utils.py | 6 +++--- freqtrade/exchange/exchange.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index eecb63d07..1459dfd78 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path, new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else - int(arrow.utcnow().shift( - days=-new_pairs_days).float_timestamp) * 1000 + arrow.utcnow().shift( + days=-new_pairs_days).int_timestamp * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -272,7 +272,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 else: - since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000 + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 235f03269..42e86db3e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -578,7 +578,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), - 'timestamp': int(arrow.utcnow().int_timestamp * 1000), + 'timestamp': arrow.utcnow().int_timestamp * 1000, 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 744b1268d..4c364bfce 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -89,7 +89,7 @@ class AgeFilter(IPairList): if len(daily_candles) >= self._min_days_listed: # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = int(arrow.utcnow().int_timestamp) * 1000 + self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 return True else: self.log_once(f"Removed {pair} from whitelist, because age " From dec523eef0c384630c325789dd8b6eae83ee2d38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 6 Jul 2021 07:20:05 +0200 Subject: [PATCH 48/48] Display verison of installed FreqUI --- freqtrade/rpc/api_server/web_ui.py | 11 +++++++++++ tests/rpc/test_rpc_apiserver.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index a8c737e04..76c8ed8f2 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -18,6 +18,17 @@ async def fallback(): return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html')) +@router_ui.get('/ui_version', include_in_schema=False) +async def ui_version(): + from freqtrade.commands.deploy_commands import read_ui_version + uibase = Path(__file__).parent / 'ui/installed/' + version = read_ui_version(uibase) + + return { + "version": version if version else "not_installed", + } + + @router_ui.get('/{rest_of_path:path}', include_in_schema=False) async def index_html(rest_of_path: str): """ diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b8dd112c9..89da68da7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -105,6 +105,15 @@ def test_api_ui_fallback(botclient): assert rc.status_code == 200 +def test_api_ui_version(botclient, mocker): + ftbot, client = botclient + + mocker.patch('freqtrade.commands.deploy_commands.read_ui_version', return_value='0.1.2') + rc = client_get(client, "/ui_version") + assert rc.status_code == 200 + assert rc.json()['version'] == '0.1.2' + + def test_api_auth(): with pytest.raises(ValueError): create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType")