diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 4c57efeae..5fa667ccc 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,10 +1,10 @@ -from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) # noqa: 401 - -from .list_commands import (start_list_exchanges, # noqa: F401 +from freqtrade.commands.data_commands import start_download_data # noqa: F401 +from freqtrade.commands.deploy_commands import (start_create_userdir, # noqa: F401 + start_new_hyperopt, start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: F401 + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 start_list_markets, start_list_strategies, start_list_timeframes) -from .utils import setup_utils_configuration # noqa: F401 -from .utils import (start_download_data, # noqa: F401 - start_test_pairlist) -from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir) # noqa: F401 -from .trade_commands import start_trading # noqa: F401 +from freqtrade.commands.trade_commands import start_trading # noqa: F401 +from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py new file mode 100644 index 000000000..b1c847d1d --- /dev/null +++ b/freqtrade/commands/data_commands.py @@ -0,0 +1,64 @@ +import logging +import sys +from typing import Any, Dict, List + +import arrow + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import (convert_trades_to_ohlcv, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {config["datadir"]}') + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config["pairs"], datadir=config['datadir'], + timerange=timerange, erase=config.get("erase")) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index ba9173cf2..a597d5335 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -1,30 +1,11 @@ -import csv import logging -import sys -from collections import OrderedDict -from operator import itemgetter -from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict -import arrow import rapidjson -from colorama import init as colorama_init -from tabulate import tabulate -from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials, +from freqtrade.configuration import (Configuration, remove_credentials, validate_config_consistency) -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) -from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) -from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -46,54 +27,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str return config -def start_download_data(args: Dict[str, Any]) -> None: - """ - Download data (former download_backtest_data.py script) - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - timerange = TimeRange() - if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {config["datadir"]}') - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) - try: - - if config.get('download_trades'): - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase")) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - else: - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - - except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - - def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration diff --git a/tests/test_utils.py b/tests/test_utils.py index c686d2117..1328e3981 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch(