From c85cd13ca1a5bdbba07659d989a3f09de9464456 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 13:31:42 +0200 Subject: [PATCH 01/25] Change default backtest result to "backtest_results" - backtest_data is misleading --- docs/backtesting.md | 2 +- docs/bot-usage.md | 4 ++-- docs/data-analysis.md | 2 +- docs/plotting.md | 2 +- freqtrade/configuration/cli_options.py | 4 ++-- user_data/backtest_results/.gitkeep | 0 6 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 user_data/backtest_results/.gitkeep diff --git a/docs/backtesting.md b/docs/backtesting.md index 179bcee15..57f9f6296 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -237,7 +237,7 @@ All listed Strategies need to be in the same directory. freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strategy-list Strategy001 Strategy002 --export trades ``` -This will save the results to `user_data/backtest_data/backtest-result-.json`, injecting the strategy-name into the target filename. +This will save the results to `user_data/backtest_results/backtest-result-.json`, injecting the strategy-name into the target filename. There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table). Detailed output for all strategies one after the other will be available, so make sure to scroll up. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 8877ed010..ef28eb60c 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -176,8 +176,8 @@ optional arguments: --export-filename PATH Save backtest results to this filename requires --export to be set as well Example --export- - filename=user_data/backtest_data/backtest_today.json - (default: user_data/backtest_data/backtest- + filename=user_data/backtest_results/backtest_today.json + (default: user_data/backtest_results/backtest- result.json) ``` diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 1940fa3e6..cda7a9a57 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -15,7 +15,7 @@ Freqtrade provides the `load_backtest_data()` helper function to easily load the ``` python from freqtrade.data.btanalysis import load_backtest_data -df = load_backtest_data("user_data/backtest-result.json") +df = load_backtest_data("user_data/backtest_results/backtest-result.json") # Show value-counts per pair df.groupby("pair")["sell_reason"].value_counts() diff --git a/docs/plotting.md b/docs/plotting.md index b8e041d61..acab60b23 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -64,7 +64,7 @@ python3 scripts/plot_dataframe.py --db-url sqlite:///tradesv3.dry_run.sqlite -p To plot trades from a backtesting result, use `--export-filename ` ``` bash -python3 scripts/plot_dataframe.py --export-filename user_data/backtest_data/backtest-result.json -p BTC/ETH +python3 scripts/plot_dataframe.py --export-filename user_data/backtest_results/backtest-result.json -p BTC/ETH ``` To plot a custom strategy the strategy should have first be backtested. diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 04fde2051..4ec280d0f 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -146,9 +146,9 @@ AVAILABLE_CLI_OPTIONS = { '--export-filename', help='Save backtest results to the file with this filename (default: `%(default)s`). ' 'Requires `--export` to be set as well. ' - 'Example: `--export-filename=user_data/backtest_data/backtest_today.json`', + 'Example: `--export-filename=user_data/backtest_results/backtest_today.json`', metavar='PATH', - default=os.path.join('user_data', 'backtest_data', + default=os.path.join('user_data', 'backtest_results', 'backtest-result.json'), ), # Edge diff --git a/user_data/backtest_results/.gitkeep b/user_data/backtest_results/.gitkeep new file mode 100644 index 000000000..e69de29bb From 6c3a0eb1d6a29df271bf0a32fa8baddec591acd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 13:42:56 +0200 Subject: [PATCH 02/25] add create_userdir function --- freqtrade/configuration/configuration.py | 2 +- ...create_datadir.py => folder_operations.py} | 14 +++++++++++++ freqtrade/tests/test_configuration.py | 20 ++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) rename freqtrade/configuration/{create_datadir.py => folder_operations.py} (56%) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 17ad37d6a..d96a291e2 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -10,7 +10,7 @@ from typing import Any, Callable, Dict, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.create_datadir import create_datadir +from freqtrade.configuration.folder_operations import create_datadir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts diff --git a/freqtrade/configuration/create_datadir.py b/freqtrade/configuration/folder_operations.py similarity index 56% rename from freqtrade/configuration/create_datadir.py rename to freqtrade/configuration/folder_operations.py index acc3a29ca..f402f91e0 100644 --- a/freqtrade/configuration/create_datadir.py +++ b/freqtrade/configuration/folder_operations.py @@ -18,3 +18,17 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str folder.mkdir(parents=True) logger.info(f'Created data directory: {datadir}') return str(folder) + + +def create_userdata_dir(directory: str): + sub_dirs = ["backtest_results", "data", "hyperopts", "plots", "strategies", ] + folder = Path(directory) + if not folder.is_dir(): + folder.mkdir(parents=True) + logger.info(f'Created user-data directory: {folder}') + + # Create required subdirectories + for f in sub_dirs: + subfolder = folder / f + if not subfolder.is_dir(): + subfolder.mkdir(parents=False) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 56ff79625..21df8d4e5 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,7 +13,7 @@ from jsonschema import Draft4Validator, ValidationError, validate from freqtrade import OperationalException, constants from freqtrade.configuration import Arguments, Configuration from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.create_datadir import create_datadir +from freqtrade.configuration.folder_operations import create_datadir, create_userdata_dir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.loggers import _set_loggers @@ -609,6 +609,24 @@ def test_create_datadir(mocker, default_conf, caplog) -> None: assert log_has('Created data directory: /foo/bar', caplog.record_tuples) +def test_create_userdata_dir(mocker, default_conf, caplog) -> None: + mocker.patch.object(Path, "is_dir", MagicMock(return_value=False)) + md = mocker.patch.object(Path, 'mkdir', MagicMock()) + + create_userdata_dir('/tmp/bar') + assert md.call_count == 6 + assert md.call_args[1]['parents'] is False + assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) + + +def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None: + mocker.patch.object(Path, "is_dir", MagicMock(return_value=True)) + md = mocker.patch.object(Path, 'mkdir', MagicMock()) + + create_userdata_dir('/tmp/bar') + assert md.call_count == 0 + + def test_validate_tsl(default_conf): default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0 From 23435512c461a0b3331314f4e06911d90fe94497 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 14:13:38 +0200 Subject: [PATCH 03/25] Add create-userdir command to initialize a user directory --- freqtrade/configuration/arguments.py | 10 ++++++++-- freqtrade/configuration/cli_options.py | 5 +++++ freqtrade/configuration/configuration.py | 9 ++++++++- freqtrade/tests/conftest.py | 1 + freqtrade/utils.py | 16 +++++++++++++++- 5 files changed, 37 insertions(+), 4 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 4f0c3d31b..0651d9544 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -31,6 +31,8 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] +ARGS_CREATE_USERDIR = ["user_data_dir"] + ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + @@ -106,8 +108,7 @@ class Arguments(object): :return: None """ from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import start_list_exchanges - + from freqtrade.utils import start_create_userdir, start_list_exchanges subparsers = self.parser.add_subparsers(dest='subparser') # Add backtesting subcommand @@ -125,6 +126,11 @@ class Arguments(object): hyperopt_cmd.set_defaults(func=start_hyperopt) self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd) + create_userdir_cmd = subparsers.add_parser('create-userdir', + help="Create user-data directory.") + create_userdir_cmd.set_defaults(func=start_create_userdir) + self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) + # Add list-exchanges subcommand list_exchanges_cmd = subparsers.add_parser( 'list-exchanges', diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 4ec280d0f..00c28ed07 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -58,6 +58,11 @@ AVAILABLE_CLI_OPTIONS = { help='Path to backtest data.', metavar='PATH', ), + "user_data_dir": Arg( + '--userdir', '--user-data-dir', + help='Path to Userdata Directory.', + metavar='PATH', + ), # Main options "strategy": Arg( '-s', '--strategy', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index d96a291e2..0fd092728 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -6,11 +6,12 @@ import logging import sys import warnings from argparse import Namespace +from pathlib import Path from typing import Any, Callable, Dict, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.folder_operations import create_datadir +from freqtrade.configuration.folder_operations import create_datadir, create_userdata_dir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts @@ -173,6 +174,12 @@ class Configuration(object): Extract information for sys.argv and load datadir configuration: the --datadir option """ + if 'user_data_dir' in self.args and self.args.user_data_dir: + config.update({'user_data_dir': self.args.user_data_dir}) + create_userdata_dir(config['user_data_dir']) + elif 'user_data_dir' not in config: + config.update({'user_data_dir': str(Path.cwd() / "user_data")}) + if 'datadir' in self.args and self.args.datadir: config.update({'datadir': create_datadir(config, self.args.datadir)}) else: diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 5862a2e89..de3b2e4d9 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -234,6 +234,7 @@ def default_conf(): }, "initial_state": "running", "db_url": "sqlite://", + "user_data_dir": "user_data", "verbosity": 3, } return configuration diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d550ef43c..53325b70b 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,12 +1,13 @@ import logging +import sys from argparse import Namespace from typing import Any, Dict from freqtrade.configuration import Configuration +from freqtrade.configuration.folder_operations import create_userdata_dir from freqtrade.exchange import available_exchanges from freqtrade.state import RunMode - logger = logging.getLogger(__name__) @@ -39,3 +40,16 @@ def start_list_exchanges(args: Namespace) -> None: else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") + + +def start_create_userdir(args: Namespace) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopts, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args.user_data_dir: + create_userdata_dir(args.user_data_dir) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) From 56c8bdbaa24b72d64d260a2dc1dcee030f65f478 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 14:32:00 +0200 Subject: [PATCH 04/25] Test create-userdir command line option --- freqtrade/tests/test_utils.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index a12b709d7..2bbece33f 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,8 +1,12 @@ -from freqtrade.utils import setup_utils_configuration, start_list_exchanges -from freqtrade.tests.conftest import get_args -from freqtrade.state import RunMode - import re +from unittest.mock import MagicMock + +import pytest + +from freqtrade.state import RunMode +from freqtrade.tests.conftest import get_args, log_has +from freqtrade.utils import (setup_utils_configuration, start_create_userdir, + start_list_exchanges) def test_setup_utils_configuration(): @@ -40,3 +44,26 @@ def test_list_exchanges(capsys): assert not re.match(r"Exchanges supported by ccxt and available.*", captured.out) assert re.search(r"^binance$", captured.out, re.MULTILINE) assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + + +def test_create_datadir_failed(caplog): + + args = [ + "create-userdir", + ] + with pytest.raises(SystemExit): + start_create_userdir(get_args(args)) + assert log_has("`create-userdir` requires --userdir to be set.", caplog.record_tuples) + + +def test_create_datadir(caplog, mocker): + cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) + args = [ + "create-userdir", + "--userdir", + "/temp/freqtrade/test" + ] + start_create_userdir(get_args(args)) + + assert cud.call_count == 1 + assert len(caplog.record_tuples) == 0 From 1b2581f0cb7c4ed9f15619533592fa9a9aad915d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 14:32:29 +0200 Subject: [PATCH 05/25] Add user_data_dir to configuration --- freqtrade/configuration/arguments.py | 2 +- freqtrade/configuration/configuration.py | 6 +++++- freqtrade/configuration/folder_operations.py | 6 ++++-- freqtrade/tests/test_configuration.py | 16 +++++++++++++--- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 0651d9544..ef19d121e 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -9,7 +9,7 @@ import arrow from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS from freqtrade import constants -ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir"] +ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] ARGS_STRATEGY = ["strategy", "strategy_path"] diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 0fd092728..78584ce6a 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -176,9 +176,13 @@ class Configuration(object): """ if 'user_data_dir' in self.args and self.args.user_data_dir: config.update({'user_data_dir': self.args.user_data_dir}) - create_userdata_dir(config['user_data_dir']) elif 'user_data_dir' not in config: + # Default to cwd/user_data (legacy option ...) config.update({'user_data_dir': str(Path.cwd() / "user_data")}) + # reset to user_data_dir so this contains the absolute path. + config['user_data_dir'] = create_userdata_dir(config['user_data_dir']) + + logger.info('Using user-data directory: %s ...', config['user_data_dir']) if 'datadir' in self.args and self.args.datadir: config.update({'datadir': create_datadir(config, self.args.datadir)}) diff --git a/freqtrade/configuration/folder_operations.py b/freqtrade/configuration/folder_operations.py index f402f91e0..d43dd1bf1 100644 --- a/freqtrade/configuration/folder_operations.py +++ b/freqtrade/configuration/folder_operations.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str: - folder = Path(datadir) if datadir else Path('user_data/data') + folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data") if not datadir: # set datadir exchange_name = config.get('exchange', {}).get('name').lower() @@ -20,7 +20,7 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str return str(folder) -def create_userdata_dir(directory: str): +def create_userdata_dir(directory: str) -> str: sub_dirs = ["backtest_results", "data", "hyperopts", "plots", "strategies", ] folder = Path(directory) if not folder.is_dir(): @@ -32,3 +32,5 @@ def create_userdata_dir(directory: str): subfolder = folder / f if not subfolder.is_dir(): subfolder.mkdir(parents=False) + # TODO: convert this to return Path + return str(folder.resolve()) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 21df8d4e5..4d1acc8d2 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -331,11 +331,15 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'freqtrade.configuration.configuration.create_datadir', lambda c, x: x ) - + mocker.patch( + 'freqtrade.configuration.configuration.create_userdata_dir', + lambda x: x + ) arglist = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', + '--userdir', "/tmp/freqtrade", 'backtesting', '--ticker-interval', '1m', '--live', @@ -357,7 +361,11 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has( - 'Using data directory: {} ...'.format(config['datadir']), + 'Using data directory: {} ...'.format("/foo/bar"), + caplog.record_tuples + ) + assert log_has( + 'Using user-data directory: {} ...'.format("/tmp/freqtrade"), caplog.record_tuples ) assert 'ticker_interval' in config @@ -613,10 +621,12 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: mocker.patch.object(Path, "is_dir", MagicMock(return_value=False)) md = mocker.patch.object(Path, 'mkdir', MagicMock()) - create_userdata_dir('/tmp/bar') + x = create_userdata_dir('/tmp/bar') assert md.call_count == 6 assert md.call_args[1]['parents'] is False assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) + assert isinstance(x, str) + assert x == "/tmp/bar" def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None: From da755d1c83389a9d242923f480fcb23ae8f05a57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 15:37:52 +0200 Subject: [PATCH 06/25] Remove obsolete variable --- scripts/download_backtest_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index ed96cec71..e721d77ba 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -20,8 +20,6 @@ import logging logger = logging.getLogger('download_backtest_data') -DEFAULT_DL_PATH = 'user_data/data' - # Do not read the default config if config is not specified # in the command line options explicitely arguments = Arguments(sys.argv[1:], 'Download backtest data', From eab82fdec7ea73fa53e8a7f348e38a0414537046 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 15:49:52 +0200 Subject: [PATCH 07/25] plot-scripts use user_data_dir --- freqtrade/configuration/folder_operations.py | 2 +- freqtrade/plot/plotting.py | 6 +++--- freqtrade/tests/test_plotting.py | 4 +++- scripts/plot_dataframe.py | 3 ++- scripts/plot_profit.py | 3 ++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration/folder_operations.py b/freqtrade/configuration/folder_operations.py index d43dd1bf1..2ea9cd268 100644 --- a/freqtrade/configuration/folder_operations.py +++ b/freqtrade/configuration/folder_operations.py @@ -33,4 +33,4 @@ def create_userdata_dir(directory: str) -> str: if not subfolder.is_dir(): subfolder.mkdir(parents=False) # TODO: convert this to return Path - return str(folder.resolve()) + return folder diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index dde6f78f0..948964462 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -308,7 +308,7 @@ def generate_plot_filename(pair, ticker_interval) -> str: return file_name -def store_plot_file(fig, filename: str, auto_open: bool = False) -> None: +def store_plot_file(fig, filename: str, folder: Path, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -317,7 +317,7 @@ def store_plot_file(fig, filename: str, auto_open: bool = False) -> None: :return: None """ - Path("user_data/plots").mkdir(parents=True, exist_ok=True) + folder.mkdir(parents=True, exist_ok=True) - plot(fig, filename=str(Path('user_data/plots').joinpath(filename)), + plot(fig, filename=str(folder.joinpath(filename)), auto_open=auto_open) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 0ebadf720..c80d0e780 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -1,5 +1,6 @@ from copy import deepcopy +from pathlib import Path from unittest.mock import MagicMock import plotly.graph_objs as go @@ -209,7 +210,8 @@ def test_generate_Plot_filename(): def test_generate_plot_file(mocker, caplog): fig = generage_empty_figure() plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) - store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html") + store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html", + folder=Path("user_data/plots")) assert plot_mock.call_count == 1 assert plot_mock.call_args[0][0] == fig diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 034a6f448..04911f93e 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -77,7 +77,8 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): indicators2=config["indicators2"].split(",") ) - store_plot_file(fig, generate_plot_filename(pair, config['ticker_interval'])) + store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), + folder=config['user_data_dir'] / "plot") logger.info('End of ploting process %s plots generated', pair_counter) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 4290bca45..c83ad1088 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -32,7 +32,8 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) - store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) + store_plot_file(fig, filename='freqtrade-profit-plot.html', + folder=config['user_data_dir'] / "plot", auto_open=True) def plot_parse_args(args: List[str]) -> Dict[str, Any]: From ae0e00118736de14cb5337174d5865c5986e4c6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 15:56:32 +0200 Subject: [PATCH 08/25] Fix some bugs in tests --- freqtrade/tests/test_configuration.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 4d1acc8d2..15a2a3271 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -305,6 +305,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config + assert 'user_data_dir' in config assert log_has( 'Using data directory: {} ...'.format(config['datadir']), caplog.record_tuples @@ -333,7 +334,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) mocker.patch( 'freqtrade.configuration.configuration.create_userdata_dir', - lambda x: x + lambda x: Path(x) ) arglist = [ '--config', 'config.json', @@ -368,6 +369,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non 'Using user-data directory: {} ...'.format("/tmp/freqtrade"), caplog.record_tuples ) + assert 'user_data_dir' in config + assert 'ticker_interval' in config assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', caplog.record_tuples) @@ -625,8 +628,8 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: assert md.call_count == 6 assert md.call_args[1]['parents'] is False assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) - assert isinstance(x, str) - assert x == "/tmp/bar" + assert isinstance(x, Path) + assert str(x) == "/tmp/bar" def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None: From 0a253d66d07c4bb4090a981a6a9a57756060c4eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 15:56:44 +0200 Subject: [PATCH 09/25] Remove os.path from hyperopt --- freqtrade/optimize/hyperopt.py | 11 +++++------ freqtrade/tests/optimize/test_hyperopt.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 759ceffbe..a1ec43c14 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,7 +5,6 @@ This module contains the hyperopt logic """ import logging -import os import sys from operator import itemgetter @@ -31,9 +30,9 @@ logger = logging.getLogger(__name__) INITIAL_POINTS = 30 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization -TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl') -TRIALSDATA_PICKLE = os.path.join('user_data', 'hyperopt_results.pickle') -HYPEROPT_LOCKFILE = os.path.join('user_data', 'hyperopt.lock') +TICKERDATA_PICKLE = Path.cwd() / 'user_data' / 'hyperopt_tickerdata.pkl' +TRIALSDATA_PICKLE = Path.cwd() / 'user_data' / 'hyperopt_results.pickle' +HYPEROPT_LOCKFILE = Path.cwd() / 'user_data' / 'hyperopt.lock' class Hyperopt(Backtesting): @@ -115,7 +114,7 @@ class Hyperopt(Backtesting): """ logger.info('Reading Trials from \'%s\'', self.trials_file) trials = load(self.trials_file) - os.remove(self.trials_file) + self.trials_file.unlink() return trials def log_trials_result(self) -> None: @@ -269,7 +268,7 @@ class Hyperopt(Backtesting): def load_previous_results(self): """ read trials file if we have one """ - if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + if self.trials_file.is_file() and self.trials_file.stat().st_size > 0: self.trials = self.read_trials() logger.info( 'Loaded %d previous evaluations from disk.', diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index fad89e877..2370e145e 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,12 +1,13 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import os from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import pandas as pd import pytest from arrow import Arrow from filelock import Timeout +from pathlib import Path from freqtrade import DependencyException from freqtrade.data.converter import parse_ticker_dataframe @@ -53,11 +54,14 @@ def create_trials(mocker, hyperopt) -> None: - we might have a pickle'd file so make sure that we return false when looking for it """ - hyperopt.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle') + hyperopt.trials_file = Path('freqtrade/tests/optimize/ut_trials.pickle') - mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False) - mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1) - mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True) + mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) + stat_mock = MagicMock() + stat_mock.st_size = PropertyMock(return_value=1) + mocker.patch.object(Path, "stat", MagicMock(return_value=False)) + + mocker.patch.object(Path, "unlink", MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) return [{'loss': 1, 'result': 'foo', 'params': {}}] From 113947132c0a6740ef6e8b19a37759a0b0715a57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 16:06:44 +0200 Subject: [PATCH 10/25] user_data_dir is PATH in config, not str --- freqtrade/tests/conftest.py | 2 +- freqtrade/tests/test_configuration.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index de3b2e4d9..0aaf28d96 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -234,7 +234,7 @@ def default_conf(): }, "initial_state": "running", "db_url": "sqlite://", - "user_data_dir": "user_data", + "user_data_dir": Path("user_data"), "verbosity": 3, } return configuration diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 15a2a3271..3d75818d4 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -53,6 +53,7 @@ def test_load_config_incorrect_stake_amount(default_conf) -> None: def test_load_config_file(default_conf, mocker, caplog) -> None: + del default_conf['user_data_dir'] file_mock = mocker.patch('freqtrade.configuration.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) From 2c7a248307356fb6170ac2aa79c7f9e5443e0066 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 16:07:06 +0200 Subject: [PATCH 11/25] Use user_data_dir in hyperopt --- freqtrade/optimize/__init__.py | 4 ++-- freqtrade/optimize/hyperopt.py | 17 ++++++++++------- freqtrade/tests/optimize/test_hyperopt.py | 9 ++++----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 8b548eefe..2c7c42c4d 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -64,14 +64,14 @@ def start_hyperopt(args: Namespace) -> None: :return: None """ # Import here to avoid loading hyperopt module when it's not used - from freqtrade.optimize.hyperopt import Hyperopt, HYPEROPT_LOCKFILE + from freqtrade.optimize.hyperopt import Hyperopt # Initialize configuration config = setup_configuration(args, RunMode.HYPEROPT) logger.info('Starting freqtrade in Hyperopt mode') - lock = FileLock(HYPEROPT_LOCKFILE) + lock = FileLock(Hyperopt.get_lock_filename(config)) try: with lock.acquire(timeout=1): diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index a1ec43c14..2fff91e58 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -30,9 +30,6 @@ logger = logging.getLogger(__name__) INITIAL_POINTS = 30 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization -TICKERDATA_PICKLE = Path.cwd() / 'user_data' / 'hyperopt_tickerdata.pkl' -TRIALSDATA_PICKLE = Path.cwd() / 'user_data' / 'hyperopt_results.pickle' -HYPEROPT_LOCKFILE = Path.cwd() / 'user_data' / 'hyperopt.lock' class Hyperopt(Backtesting): @@ -50,6 +47,8 @@ class Hyperopt(Backtesting): self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function + self.trials_file = self.config['user_data_dir'] / 'hyperopt_results.pickle' + self.tickerdata_pickle = self.config['user_data_dir'] / 'hyperopt_tickerdata.pkl' self.total_tries = config.get('epochs', 0) self.current_best_loss = 100 @@ -59,7 +58,6 @@ class Hyperopt(Backtesting): logger.info("Continuing on previous hyperopt results.") # Previous evaluations - self.trials_file = TRIALSDATA_PICKLE self.trials: List = [] # Populate functions here (hasattr is slow so should not be run during "regular" operations) @@ -77,11 +75,16 @@ class Hyperopt(Backtesting): self.max_open_trades = 0 self.position_stacking = self.config.get('position_stacking', False), + @staticmethod + def get_lock_filename(config) -> str: + + return str(config['user_data_dir'] / 'hyperopt.lock') + def clean_hyperopt(self): """ Remove hyperopt pickle files to restart hyperopt. """ - for f in [TICKERDATA_PICKLE, TRIALSDATA_PICKLE]: + for f in [self.tickerdata_pickle, self.trials_file]: p = Path(f) if p.is_file(): logger.info(f"Removing `{p}`.") @@ -199,7 +202,7 @@ class Hyperopt(Backtesting): if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] - processed = load(TICKERDATA_PICKLE) + processed = load(self.tickerdata_pickle) min_date, max_date = get_timeframe(processed) @@ -305,7 +308,7 @@ class Hyperopt(Backtesting): preprocessed = self.strategy.tickerdata_to_dataframe(data) - dump(preprocessed, TICKERDATA_PICKLE) + dump(preprocessed, self.tickerdata_pickle) # We don't need exchange instance anymore while running hyperopt self.exchange = None # type: ignore diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 2370e145e..dc34198cf 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -15,8 +15,7 @@ from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.optimize.default_hyperopt import DefaultHyperOpts from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss -from freqtrade.optimize.hyperopt import (HYPEROPT_LOCKFILE, TICKERDATA_PICKLE, - Hyperopt) +from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver, HyperOptLossResolver from freqtrade.state import RunMode from freqtrade.strategy.interface import SellType @@ -272,7 +271,7 @@ def test_start_failure(mocker, default_conf, caplog) -> None: def test_start_filelock(mocker, default_conf, caplog) -> None: - start_mock = MagicMock(side_effect=Timeout(HYPEROPT_LOCKFILE)) + start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(default_conf))) patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) patch_exchange(mocker) @@ -609,10 +608,10 @@ def test_clean_hyperopt(mocker, default_conf, caplog): }) mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) - Hyperopt(default_conf) + h = Hyperopt(default_conf) assert unlinkmock.call_count == 2 - assert log_has(f"Removing `{TICKERDATA_PICKLE}`.", caplog.record_tuples) + assert log_has(f"Removing `{h.tickerdata_pickle}`.", caplog.record_tuples) def test_continue_hyperopt(mocker, default_conf, caplog): From 432b106d58182688606a558de62f1899f1394e0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 Jul 2019 16:19:31 +0200 Subject: [PATCH 12/25] Improve docstring, remove unneeded method --- freqtrade/configuration/configuration.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 78584ce6a..a957e6e2a 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -124,7 +124,9 @@ class Configuration(object): setup_logging(config) - def _process_strategy_options(self, config: Dict[str, Any]) -> None: + def _process_common_options(self, config: Dict[str, Any]) -> None: + + self._process_logging_options(config) # Set strategy if not specified in config and or if it's non default if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): @@ -133,11 +135,6 @@ class Configuration(object): self._args_to_config(config, argname='strategy_path', logstring='Using additional Strategy lookup path: {}') - def _process_common_options(self, config: Dict[str, Any]) -> None: - - self._process_logging_options(config) - self._process_strategy_options(config) - if ('db_url' in self.args and self.args.db_url and self.args.db_url != constants.DEFAULT_DB_PROD_URL): config.update({'db_url': self.args.db_url}) @@ -171,8 +168,8 @@ class Configuration(object): def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ - Extract information for sys.argv and load datadir configuration: - the --datadir option + Extract information for sys.argv and load directory configurations + --user-data, --datadir """ if 'user_data_dir' in self.args and self.args.user_data_dir: config.update({'user_data_dir': self.args.user_data_dir}) @@ -181,7 +178,6 @@ class Configuration(object): config.update({'user_data_dir': str(Path.cwd() / "user_data")}) # reset to user_data_dir so this contains the absolute path. config['user_data_dir'] = create_userdata_dir(config['user_data_dir']) - logger.info('Using user-data directory: %s ...', config['user_data_dir']) if 'datadir' in self.args and self.args.datadir: From 9de8d7276e170325a0cb75442b131dd95e12a1f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 25 Jul 2019 07:16:52 +0200 Subject: [PATCH 13/25] have strategyresolver use user_data_dir --- freqtrade/resolvers/iresolver.py | 4 ++-- freqtrade/resolvers/strategy_resolver.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 9d0c97917..5a62d448f 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -57,7 +57,7 @@ class IResolver(object): if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue - module_path = Path.resolve(directory.joinpath(entry)) + module_path = entry.resolve() obj = IResolver._get_valid_object( object_type, module_path, object_name ) @@ -84,6 +84,6 @@ class IResolver(object): f"from '{module_path}'...") return module except FileNotFoundError: - logger.warning('Path "%s" does not exist.', _path.relative_to(Path.cwd())) + logger.warning('Path "%s" does not exist.', _path.resolve()) return None diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index aa73327ff..37aa96b68 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -123,7 +123,7 @@ class StrategyResolver(IResolver): current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() abs_paths = [ - Path.cwd().joinpath('user_data/strategies'), + config['user_data_dir'].joinpath('strategies'), current_path, ] From 333413d29846b4f505f580096ba1b916e7b0006f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 25 Jul 2019 07:17:25 +0200 Subject: [PATCH 14/25] Add default_conf to strategy tests --- freqtrade/tests/strategy/test_strategy.py | 166 +++++++++++----------- 1 file changed, 85 insertions(+), 81 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index df8c0f126..c5fcd46d9 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -64,15 +64,18 @@ def test_search_strategy(): assert s is None -def test_load_strategy(result): - resolver = StrategyResolver({'strategy': 'TestStrategy'}) +def test_load_strategy(default_conf, result): + default_conf.update({'strategy': 'TestStrategy'}) + resolver = StrategyResolver(default_conf) assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) -def test_load_strategy_base64(result, caplog): +def test_load_strategy_base64(result, caplog, default_conf): with open("user_data/strategies/test_strategy.py", "rb") as file: encoded_string = urlsafe_b64encode(file.read()).decode("utf-8") - resolver = StrategyResolver({'strategy': 'TestStrategy:{}'.format(encoded_string)}) + default_conf.update({'strategy': 'TestStrategy:{}'.format(encoded_string)}) + + resolver = StrategyResolver(default_conf) assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) # Make sure strategy was loaded from base64 (using temp directory)!! assert log_has_re(r"Using resolved strategy TestStrategy from '" @@ -80,47 +83,47 @@ def test_load_strategy_base64(result, caplog): caplog.record_tuples) -def test_load_strategy_invalid_directory(result, caplog): - resolver = StrategyResolver() +def test_load_strategy_invalid_directory(result, caplog, default_conf): + resolver = StrategyResolver(default_conf) extra_dir = Path.cwd() / 'some/path' - resolver._load_strategy('TestStrategy', config={}, extra_dir=extra_dir) + resolver._load_strategy('TestStrategy', config=default_conf, extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog.record_tuples) assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) -def test_load_not_found_strategy(): - strategy = StrategyResolver() +def test_load_not_found_strategy(default_conf): + strategy = StrategyResolver(default_conf) with pytest.raises(OperationalException, match=r"Impossible to load Strategy 'NotFoundStrategy'. " r"This class does not exist or contains Python code errors."): - strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) + strategy._load_strategy(strategy_name='NotFoundStrategy', config=default_conf) -def test_load_staticmethod_importerror(mocker, caplog): +def test_load_staticmethod_importerror(mocker, caplog, default_conf): mocker.patch("freqtrade.resolvers.strategy_resolver.import_strategy", Mock( side_effect=TypeError("can't pickle staticmethod objects"))) with pytest.raises(OperationalException, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"This class does not exist or contains Python code errors."): - StrategyResolver() + StrategyResolver(default_conf) assert log_has_re(r".*Error: can't pickle staticmethod objects", caplog.record_tuples) -def test_strategy(result): - config = {'strategy': 'DefaultStrategy'} +def test_strategy(result, default_conf): + default_conf.update({'strategy': 'DefaultStrategy'}) - resolver = StrategyResolver(config) + resolver = StrategyResolver(default_conf) metadata = {'pair': 'ETH/BTC'} assert resolver.strategy.minimal_roi[0] == 0.04 - assert config["minimal_roi"]['0'] == 0.04 + assert default_conf["minimal_roi"]['0'] == 0.04 assert resolver.strategy.stoploss == -0.10 - assert config['stoploss'] == -0.10 + assert default_conf['stoploss'] == -0.10 assert resolver.strategy.ticker_interval == '5m' - assert config['ticker_interval'] == '5m' + assert default_conf['ticker_interval'] == '5m' df_indicators = resolver.strategy.advise_indicators(result, metadata=metadata) assert 'adx' in df_indicators @@ -132,15 +135,15 @@ def test_strategy(result): assert 'sell' in dataframe.columns -def test_strategy_override_minimal_roi(caplog): +def test_strategy_override_minimal_roi(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'minimal_roi': { "0": 0.5 } - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.minimal_roi[0] == 0.5 assert ('freqtrade.resolvers.strategy_resolver', @@ -149,13 +152,13 @@ def test_strategy_override_minimal_roi(caplog): ) in caplog.record_tuples -def test_strategy_override_stoploss(caplog): +def test_strategy_override_stoploss(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'stoploss': -0.5 - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.stoploss == -0.5 assert ('freqtrade.resolvers.strategy_resolver', @@ -164,13 +167,13 @@ def test_strategy_override_stoploss(caplog): ) in caplog.record_tuples -def test_strategy_override_trailing_stop(caplog): +def test_strategy_override_trailing_stop(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'trailing_stop': True - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.trailing_stop assert isinstance(resolver.strategy.trailing_stop, bool) @@ -180,15 +183,15 @@ def test_strategy_override_trailing_stop(caplog): ) in caplog.record_tuples -def test_strategy_override_trailing_stop_positive(caplog): +def test_strategy_override_trailing_stop_positive(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'trailing_stop_positive': -0.1, 'trailing_stop_positive_offset': -0.2 - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.trailing_stop_positive == -0.1 assert ('freqtrade.resolvers.strategy_resolver', @@ -203,15 +206,15 @@ def test_strategy_override_trailing_stop_positive(caplog): ) in caplog.record_tuples -def test_strategy_override_ticker_interval(caplog): +def test_strategy_override_ticker_interval(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'ticker_interval': 60, 'stake_currency': 'ETH' - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.ticker_interval == 60 assert resolver.strategy.stake_currency == 'ETH' @@ -221,14 +224,14 @@ def test_strategy_override_ticker_interval(caplog): ) in caplog.record_tuples -def test_strategy_override_process_only_new_candles(caplog): +def test_strategy_override_process_only_new_candles(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'process_only_new_candles': True - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.process_only_new_candles assert ('freqtrade.resolvers.strategy_resolver', @@ -237,7 +240,7 @@ def test_strategy_override_process_only_new_candles(caplog): ) in caplog.record_tuples -def test_strategy_override_order_types(caplog): +def test_strategy_override_order_types(caplog, default_conf): caplog.set_level(logging.INFO) order_types = { @@ -246,12 +249,11 @@ def test_strategy_override_order_types(caplog): 'stoploss': 'limit', 'stoploss_on_exchange': True, } - - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'order_types': order_types - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.order_types for method in ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']: @@ -264,18 +266,18 @@ def test_strategy_override_order_types(caplog): " 'stoploss_on_exchange': True}." ) in caplog.record_tuples - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'order_types': {'buy': 'market'} - } + }) # Raise error for invalid configuration with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-types mapping is incomplete."): - StrategyResolver(config) + StrategyResolver(default_conf) -def test_strategy_override_order_tif(caplog): +def test_strategy_override_order_tif(caplog, default_conf): caplog.set_level(logging.INFO) order_time_in_force = { @@ -283,11 +285,11 @@ def test_strategy_override_order_tif(caplog): 'sell': 'gtc', } - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'order_time_in_force': order_time_in_force - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.order_time_in_force for method in ['buy', 'sell']: @@ -299,36 +301,36 @@ def test_strategy_override_order_tif(caplog): " {'buy': 'fok', 'sell': 'gtc'}." ) in caplog.record_tuples - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'order_time_in_force': {'buy': 'fok'} - } + }) # Raise error for invalid configuration with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-time-in-force mapping is incomplete."): - StrategyResolver(config) + StrategyResolver(default_conf) -def test_strategy_override_use_sell_signal(caplog): +def test_strategy_override_use_sell_signal(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert not resolver.strategy.use_sell_signal assert isinstance(resolver.strategy.use_sell_signal, bool) # must be inserted to configuration - assert 'use_sell_signal' in config['experimental'] - assert not config['experimental']['use_sell_signal'] + assert 'use_sell_signal' in default_conf['experimental'] + assert not default_conf['experimental']['use_sell_signal'] - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'experimental': { 'use_sell_signal': True, }, - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.use_sell_signal assert isinstance(resolver.strategy.use_sell_signal, bool) @@ -338,25 +340,25 @@ def test_strategy_override_use_sell_signal(caplog): ) in caplog.record_tuples -def test_strategy_override_use_sell_profit_only(caplog): +def test_strategy_override_use_sell_profit_only(caplog, default_conf): caplog.set_level(logging.INFO) - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert not resolver.strategy.sell_profit_only assert isinstance(resolver.strategy.sell_profit_only, bool) # must be inserted to configuration - assert 'sell_profit_only' in config['experimental'] - assert not config['experimental']['sell_profit_only'] + assert 'sell_profit_only' in default_conf['experimental'] + assert not default_conf['experimental']['sell_profit_only'] - config = { + default_conf.update({ 'strategy': 'DefaultStrategy', 'experimental': { 'sell_profit_only': True, }, - } - resolver = StrategyResolver(config) + }) + resolver = StrategyResolver(default_conf) assert resolver.strategy.sell_profit_only assert isinstance(resolver.strategy.sell_profit_only, bool) @@ -367,10 +369,11 @@ def test_strategy_override_use_sell_profit_only(caplog): @pytest.mark.filterwarnings("ignore:deprecated") -def test_deprecate_populate_indicators(result): +def test_deprecate_populate_indicators(result, default_conf): default_location = path.join(path.dirname(path.realpath(__file__))) - resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', - 'strategy_path': default_location}) + default_conf.update({'strategy': 'TestStrategyLegacy', + 'strategy_path': default_location}) + resolver = StrategyResolver(default_conf) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") @@ -400,10 +403,11 @@ def test_deprecate_populate_indicators(result): @pytest.mark.filterwarnings("ignore:deprecated") -def test_call_deprecated_function(result, monkeypatch): +def test_call_deprecated_function(result, monkeypatch, default_conf): default_location = path.join(path.dirname(path.realpath(__file__))) - resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', - 'strategy_path': default_location}) + default_conf.update({'strategy': 'TestStrategyLegacy', + 'strategy_path': default_location}) + resolver = StrategyResolver(default_conf) metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function From a3c605f147d6c88dd027d2ba4f4b2758a75370ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Jul 2019 14:49:34 +0200 Subject: [PATCH 15/25] PairListResovler to use user_data_dir --- freqtrade/resolvers/pairlist_resolver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 3d95c0295..2c88f27a3 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -25,11 +25,11 @@ class PairListResolver(IResolver): Load the custom class from config parameter :param config: configuration dictionary or None """ - self.pairlist = self._load_pairlist(pairlist_name, kwargs={'freqtrade': freqtrade, - 'config': config}) + self.pairlist = self._load_pairlist(pairlist_name, config, kwargs={'freqtrade': freqtrade, + 'config': config}) def _load_pairlist( - self, pairlist_name: str, kwargs: dict) -> IPairList: + self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: """ Search and loads the specified pairlist. :param pairlist_name: name of the module to import @@ -39,7 +39,7 @@ class PairListResolver(IResolver): current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() abs_paths = [ - Path.cwd().joinpath('user_data/pairlist'), + config['user_data_dir'].joinpath('user_data/pairlist'), current_path, ] From 14b43b504b8e672e7b7be17c91b2bb1e06880232 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Jul 2019 15:02:11 +0200 Subject: [PATCH 16/25] Use user_data_dir for hyperopt --- freqtrade/resolvers/hyperopt_resolver.py | 18 +++++++++++------- freqtrade/resolvers/pairlist_resolver.py | 3 ++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 5027d7ddf..15d1997ef 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -31,7 +31,8 @@ class HyperOptResolver(IResolver): # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT - self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + self.hyperopt = self._load_hyperopt(hyperopt_name, config, + extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt self.hyperopt.__class__.ticker_interval = str(config['ticker_interval']) @@ -44,17 +45,18 @@ class HyperOptResolver(IResolver): "Using populate_sell_trend from DefaultStrategy.") def _load_hyperopt( - self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: + self, hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: """ Search and loads the specified hyperopt. :param hyperopt_name: name of the module to import + :param config: configuration dictionary :param extra_dir: additional directory to search for the given hyperopt :return: HyperOpt instance or None """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() abs_paths = [ - Path.cwd().joinpath('user_data/hyperopts'), + config['user_data_dir'].joinpath('hyperopts'), current_path, ] @@ -79,7 +81,7 @@ class HyperOptLossResolver(IResolver): __slots__ = ['hyperoptloss'] - def __init__(self, config: Optional[Dict] = None) -> None: + def __init__(self, config: Dict = None) -> None: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -89,7 +91,7 @@ class HyperOptLossResolver(IResolver): # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt hyperopt_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS self.hyperoptloss = self._load_hyperoptloss( - hyperopt_name, extra_dir=config.get('hyperopt_path')) + hyperopt_name, config, extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt self.hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) @@ -99,17 +101,19 @@ class HyperOptLossResolver(IResolver): f"Found hyperopt {hyperopt_name} does not implement `hyperopt_loss_function`.") def _load_hyperoptloss( - self, hyper_loss_name: str, extra_dir: Optional[str] = None) -> IHyperOptLoss: + self, hyper_loss_name: str, config: Dict, + extra_dir: Optional[str] = None) -> IHyperOptLoss: """ Search and loads the specified hyperopt loss class. :param hyper_loss_name: name of the module to import + :param config: configuration dictionary :param extra_dir: additional directory to search for the given hyperopt :return: HyperOptLoss instance or None """ current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() abs_paths = [ - Path.cwd().joinpath('user_data/hyperopts'), + config['user_data_dir'].joinpath('hyperopts'), current_path, ] diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 2c88f27a3..f38253155 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -33,13 +33,14 @@ class PairListResolver(IResolver): """ Search and loads the specified pairlist. :param pairlist_name: name of the module to import + :param config: configuration dictionary :param extra_dir: additional directory to search for the given pairlist :return: PairList instance or None """ current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() abs_paths = [ - config['user_data_dir'].joinpath('user_data/pairlist'), + config['user_data_dir'].joinpath('pairlist'), current_path, ] From 73ac98da8053202a39822f7b01d98f276045d03a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Jul 2019 15:11:41 +0200 Subject: [PATCH 17/25] Small fixes while tsting --- freqtrade/configuration/configuration.py | 1 + freqtrade/configuration/folder_operations.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index a957e6e2a..cb383d8b6 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -176,6 +176,7 @@ class Configuration(object): elif 'user_data_dir' not in config: # Default to cwd/user_data (legacy option ...) config.update({'user_data_dir': str(Path.cwd() / "user_data")}) + # reset to user_data_dir so this contains the absolute path. config['user_data_dir'] = create_userdata_dir(config['user_data_dir']) logger.info('Using user-data directory: %s ...', config['user_data_dir']) diff --git a/freqtrade/configuration/folder_operations.py b/freqtrade/configuration/folder_operations.py index 2ea9cd268..fc516a8cc 100644 --- a/freqtrade/configuration/folder_operations.py +++ b/freqtrade/configuration/folder_operations.py @@ -20,8 +20,8 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str return str(folder) -def create_userdata_dir(directory: str) -> str: - sub_dirs = ["backtest_results", "data", "hyperopts", "plots", "strategies", ] +def create_userdata_dir(directory: str) -> Path: + sub_dirs = ["backtest_results", "data", "hyperopts", "plot", "strategies", ] folder = Path(directory) if not folder.is_dir(): folder.mkdir(parents=True) @@ -32,5 +32,4 @@ def create_userdata_dir(directory: str) -> str: subfolder = folder / f if not subfolder.is_dir(): subfolder.mkdir(parents=False) - # TODO: convert this to return Path return folder From c1bc1e31373f2159c650a1f0d7580f0ddf26f675 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Jul 2019 15:34:49 +0200 Subject: [PATCH 18/25] Add documentation for user_data_dir --- docs/bot-usage.md | 50 ++++++++++++++++++++++++++++++++----------- docs/configuration.md | 1 + 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index ef28eb60c..f407e6a23 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -9,38 +9,43 @@ This page explains the different parameters of the bot and how to run it. ## Bot commands ``` -usage: freqtrade [-h] [-v] [--logfile FILE] [--version] [-c PATH] [-d PATH] - [-s NAME] [--strategy-path PATH] [--db-url PATH] - [--sd-notify] - {backtesting,edge,hyperopt} ... +usage: freqtrade [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] + [--userdir PATH] [-s NAME] [--strategy-path PATH] + [--db-url PATH] [--sd-notify] + {backtesting,edge,hyperopt,create-userdir,list-exchanges} ... Free, open source crypto trading bot positional arguments: - {backtesting,edge,hyperopt} + {backtesting,edge,hyperopt,create-userdir,list-exchanges} backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. + create-userdir Create user-data directory. + list-exchanges Print available exchanges. optional arguments: -h, --help show this help message and exit -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified + --logfile FILE Log to the file specified. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: None). Multiple - --config options may be used. Can be set to '-' to - read config from stdin. + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. -d PATH, --datadir PATH Path to backtest data. + --userdir PATH, --user-data-dir PATH + Path to Userdata Directory. -s NAME, --strategy NAME Specify strategy class name (default: - DefaultStrategy). + `DefaultStrategy`). --strategy-path PATH Specify additional strategy lookup path. - --db-url PATH Override trades database URL, this is useful if - dry_run is enabled or in custom deployments (default: - None). + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite://` for Dry Run). --sd-notify Notify systemd service manager. + ``` ### How to use a different configuration file? @@ -82,6 +87,25 @@ of your configuration in the project issues or in the Internet. See more details on this technique with examples in the documentation page on [configuration](configuration.md). +### Where to store custom data + +Freqtrade allows the creation of a user-data directory using `freqtrade create-userdir --userdir someDirectory`. +This directory will look as follows: + +user_data/ +├── backtest_results +├── data +├── hyperopts +├── plot +└── strategies + +You can add the entry "user_data_dir" to your configuration, to always point your bot to this folder. +Alternatively, pass in `--userdir` to every command. + +This directory should contain your custom strategies, custom hyperopts, backtest data (downloaded using either backtesting or the download script) and plot outputs. + +It is reccomendet to use version control to keep track of changes to your strategies. + ### How to use **--strategy**? This parameter will allow you to load your custom strategy class. diff --git a/docs/configuration.md b/docs/configuration.md index f8dbbbbbb..34d0e110f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,6 +71,7 @@ Mandatory Parameters are marked as **Required**. | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. +| `user_data_dir` | cwd()/user_data | Folder containing user data. Defaults to `./user_data/`. ### Parameters in the strategy From 03e60b9ea490d852bea26797a575d82dd6d49b7d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jul 2019 06:15:49 +0200 Subject: [PATCH 19/25] Rename folder_Operations to directory_operations --- freqtrade/configuration/configuration.py | 2 +- .../{folder_operations.py => directory_operations.py} | 0 freqtrade/tests/test_configuration.py | 2 +- freqtrade/utils.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename freqtrade/configuration/{folder_operations.py => directory_operations.py} (100%) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cb383d8b6..e56c2c06a 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -11,7 +11,7 @@ from typing import Any, Callable, Dict, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.folder_operations import create_datadir, create_userdata_dir +from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts diff --git a/freqtrade/configuration/folder_operations.py b/freqtrade/configuration/directory_operations.py similarity index 100% rename from freqtrade/configuration/folder_operations.py rename to freqtrade/configuration/directory_operations.py diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 3d75818d4..2b967a859 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,7 +13,7 @@ from jsonschema import Draft4Validator, ValidationError, validate from freqtrade import OperationalException, constants from freqtrade.configuration import Arguments, Configuration from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.folder_operations import create_datadir, create_userdata_dir +from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.loggers import _set_loggers diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 53325b70b..fa6bc2d1d 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -4,7 +4,7 @@ from argparse import Namespace from typing import Any, Dict from freqtrade.configuration import Configuration -from freqtrade.configuration.folder_operations import create_userdata_dir +from freqtrade.configuration.directory_operations import create_userdata_dir from freqtrade.exchange import available_exchanges from freqtrade.state import RunMode From 0488525888d391a651fd4dcb60ded8c799a4c6ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Jul 2019 06:49:25 +0200 Subject: [PATCH 20/25] Fix some documentation errors --- docs/bot-usage.md | 12 +++++++----- freqtrade/configuration/cli_options.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index f407e6a23..1b66cd840 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -34,9 +34,9 @@ optional arguments: Multiple --config options may be used. Can be set to `-` to read config from stdin. -d PATH, --datadir PATH - Path to backtest data. + Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH - Path to Userdata Directory. + Path to userdata directory. -s NAME, --strategy NAME Specify strategy class name (default: `DefaultStrategy`). @@ -92,19 +92,21 @@ See more details on this technique with examples in the documentation page on Freqtrade allows the creation of a user-data directory using `freqtrade create-userdir --userdir someDirectory`. This directory will look as follows: +``` user_data/ ├── backtest_results ├── data ├── hyperopts ├── plot └── strategies +``` -You can add the entry "user_data_dir" to your configuration, to always point your bot to this folder. +You can add the entry "user_data_dir" setting to your configuration, to always point your bot to this directory. Alternatively, pass in `--userdir` to every command. -This directory should contain your custom strategies, custom hyperopts, backtest data (downloaded using either backtesting or the download script) and plot outputs. +This directory should contain your custom strategies, custom hyperopts and hyperopt loss functions, backtesting historical data (downloaded using either backtesting command or the download script) and plot outputs. -It is reccomendet to use version control to keep track of changes to your strategies. +It is recommended to use version control to keep track of changes to your strategies. ### How to use **--strategy**? diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 00c28ed07..438e6c5bc 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -55,12 +55,12 @@ AVAILABLE_CLI_OPTIONS = { ), "datadir": Arg( '-d', '--datadir', - help='Path to backtest data.', + help='Path to directory with historical backtesting data.', metavar='PATH', ), "user_data_dir": Arg( '--userdir', '--user-data-dir', - help='Path to Userdata Directory.', + help='Path to userdata directory.', metavar='PATH', ), # Main options From c3d14ab9b93071a0167a544d7e2cf4a0bf2ae282 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Jul 2019 06:54:45 +0200 Subject: [PATCH 21/25] don't use "folder" ... --- docs/configuration.md | 2 +- freqtrade/plot/plotting.py | 6 +++--- scripts/plot_dataframe.py | 2 +- scripts/plot_profit.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 34d0e110f..dfe191b10 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,7 +71,7 @@ Mandatory Parameters are marked as **Required**. | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. -| `user_data_dir` | cwd()/user_data | Folder containing user data. Defaults to `./user_data/`. +| `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. ### Parameters in the strategy diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 948964462..f2c999369 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -308,7 +308,7 @@ def generate_plot_filename(pair, ticker_interval) -> str: return file_name -def store_plot_file(fig, filename: str, folder: Path, auto_open: bool = False) -> None: +def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -317,7 +317,7 @@ def store_plot_file(fig, filename: str, folder: Path, auto_open: bool = False) - :return: None """ - folder.mkdir(parents=True, exist_ok=True) + directory.mkdir(parents=True, exist_ok=True) - plot(fig, filename=str(folder.joinpath(filename)), + plot(fig, filename=str(directory.joinpath(filename)), auto_open=auto_open) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 04911f93e..e1548754a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -78,7 +78,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): ) store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), - folder=config['user_data_dir'] / "plot") + directory=config['user_data_dir'] / "plot") logger.info('End of ploting process %s plots generated', pair_counter) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index c83ad1088..578ddf15f 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -33,7 +33,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # this could be useful to gauge the overall market trend fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) store_plot_file(fig, filename='freqtrade-profit-plot.html', - folder=config['user_data_dir'] / "plot", auto_open=True) + directory=config['user_data_dir'] / "plot", auto_open=True) def plot_parse_args(args: List[str]) -> Dict[str, Any]: From 5d22d541f21e67045ca2c125f88d5e1f9652b653 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Jul 2019 06:58:26 +0200 Subject: [PATCH 22/25] Add forgotten directory --- freqtrade/tests/test_plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index c80d0e780..83a561110 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -211,7 +211,7 @@ def test_generate_plot_file(mocker, caplog): fig = generage_empty_figure() plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html", - folder=Path("user_data/plots")) + directory=Path("user_data/plots")) assert plot_mock.call_count == 1 assert plot_mock.call_args[0][0] == fig From 8cef567abcfea2fd3a1e0fac09e62f411696f817 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Jul 2019 07:07:46 +0200 Subject: [PATCH 23/25] create and use hyperopt-results folder --- docs/bot-usage.md | 1 + freqtrade/configuration/directory_operations.py | 2 +- freqtrade/optimize/hyperopt.py | 6 ++++-- freqtrade/tests/test_configuration.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 1b66cd840..130c55518 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -97,6 +97,7 @@ user_data/ ├── backtest_results ├── data ├── hyperopts +├── hyperopts_results ├── plot └── strategies ``` diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index fc516a8cc..7542c2b80 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -21,7 +21,7 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str def create_userdata_dir(directory: str) -> Path: - sub_dirs = ["backtest_results", "data", "hyperopts", "plot", "strategies", ] + sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "plot", "strategies", ] folder = Path(directory) if not folder.is_dir(): folder.mkdir(parents=True) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2fff91e58..ecf14def9 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -47,8 +47,10 @@ class Hyperopt(Backtesting): self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - self.trials_file = self.config['user_data_dir'] / 'hyperopt_results.pickle' - self.tickerdata_pickle = self.config['user_data_dir'] / 'hyperopt_tickerdata.pkl' + self.trials_file = (self.config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + self.tickerdata_pickle = (self.config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_tries = config.get('epochs', 0) self.current_best_loss = 100 diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 2b967a859..4dd3760db 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -626,7 +626,7 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: md = mocker.patch.object(Path, 'mkdir', MagicMock()) x = create_userdata_dir('/tmp/bar') - assert md.call_count == 6 + assert md.call_count == 7 assert md.call_args[1]['parents'] is False assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) assert isinstance(x, Path) From 2a141af42e538c825c28bc51c9cd303eb3c9feea Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 Jul 2019 19:39:54 +0200 Subject: [PATCH 24/25] Only create userdir when explicitly requested --- docs/bot-usage.md | 1 + freqtrade/configuration/configuration.py | 2 +- .../configuration/directory_operations.py | 21 ++++++++++++++++--- freqtrade/tests/test_configuration.py | 13 ++++++++++-- freqtrade/utils.py | 2 +- 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 130c55518..647531d3a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -104,6 +104,7 @@ user_data/ You can add the entry "user_data_dir" setting to your configuration, to always point your bot to this directory. Alternatively, pass in `--userdir` to every command. +The bot will fail to start if the directory does not exist, but will create necessary subdirectories. This directory should contain your custom strategies, custom hyperopts and hyperopt loss functions, backtesting historical data (downloaded using either backtesting command or the download script) and plot outputs. diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e56c2c06a..a2ce54e8b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -178,7 +178,7 @@ class Configuration(object): config.update({'user_data_dir': str(Path.cwd() / "user_data")}) # reset to user_data_dir so this contains the absolute path. - config['user_data_dir'] = create_userdata_dir(config['user_data_dir']) + config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False) logger.info('Using user-data directory: %s ...', config['user_data_dir']) if 'datadir' in self.args and self.args.datadir: diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 7542c2b80..395accd90 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -2,6 +2,7 @@ import logging from typing import Any, Dict, Optional from pathlib import Path +from freqtrade import OperationalException logger = logging.getLogger(__name__) @@ -20,12 +21,26 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str return str(folder) -def create_userdata_dir(directory: str) -> Path: +def create_userdata_dir(directory: str, create_dir=False) -> Path: + """ + Create userdata directory structure. + if create_dir is True, then the parent-directory will be created if it does not exist. + Sub-directories will always be created if the parent directory exists. + Raises OperationalException if given a non-existing directory. + :param directory: Directory to check + :param create_dir: Create directory if it does not exist. + :return: Path object containing the directory + """ sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "plot", "strategies", ] folder = Path(directory) if not folder.is_dir(): - folder.mkdir(parents=True) - logger.info(f'Created user-data directory: {folder}') + if create_dir: + folder.mkdir(parents=True) + logger.info(f'Created user-data directory: {folder}') + else: + raise OperationalException( + f"Directory `{folder}` does not exist. " + "Please use `freqtrade create-userdir` to create a user directory") # Create required subdirectories for f in sub_dirs: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 4dd3760db..838578e25 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -335,7 +335,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) mocker.patch( 'freqtrade.configuration.configuration.create_userdata_dir', - lambda x: Path(x) + lambda x, *args, **kwargs: Path(x) ) arglist = [ '--config', 'config.json', @@ -625,7 +625,7 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: mocker.patch.object(Path, "is_dir", MagicMock(return_value=False)) md = mocker.patch.object(Path, 'mkdir', MagicMock()) - x = create_userdata_dir('/tmp/bar') + x = create_userdata_dir('/tmp/bar', create_dir=True) assert md.call_count == 7 assert md.call_args[1]['parents'] is False assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) @@ -641,6 +641,15 @@ def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None: assert md.call_count == 0 +def test_create_userdata_dir_exists_exception(mocker, default_conf, caplog) -> None: + mocker.patch.object(Path, "is_dir", MagicMock(return_value=False)) + md = mocker.patch.object(Path, 'mkdir', MagicMock()) + + with pytest.raises(OperationalException, match=r'Directory `/tmp/bar` does not exist.*'): + create_userdata_dir('/tmp/bar', create_dir=False) + assert md.call_count == 0 + + def test_validate_tsl(default_conf): default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0 diff --git a/freqtrade/utils.py b/freqtrade/utils.py index fa6bc2d1d..2c7902ebd 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -49,7 +49,7 @@ def start_create_userdir(args: Namespace) -> None: :return: None """ if "user_data_dir" in args and args.user_data_dir: - create_userdata_dir(args.user_data_dir) + create_userdata_dir(args.user_data_dir, create_dir=True) else: logger.warning("`create-userdir` requires --userdir to be set.") sys.exit(1) From acf1e734ecb71ad79a0bcc84dc55964d36530a4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 15:09:44 +0200 Subject: [PATCH 25/25] Adapt lg_has calls to new standard --- freqtrade/tests/test_configuration.py | 2 +- freqtrade/tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 083be3e6e..9641d0a51 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -632,7 +632,7 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: x = create_userdata_dir('/tmp/bar', create_dir=True) assert md.call_count == 7 assert md.call_args[1]['parents'] is False - assert log_has('Created user-data directory: /tmp/bar', caplog.record_tuples) + assert log_has('Created user-data directory: /tmp/bar', caplog) assert isinstance(x, Path) assert str(x) == "/tmp/bar" diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index 2bbece33f..8e364f3e5 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -53,7 +53,7 @@ def test_create_datadir_failed(caplog): ] with pytest.raises(SystemExit): start_create_userdir(get_args(args)) - assert log_has("`create-userdir` requires --userdir to be set.", caplog.record_tuples) + assert log_has("`create-userdir` requires --userdir to be set.", caplog) def test_create_datadir(caplog, mocker):