diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ddcae89ab..2c01f9f48 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -155,11 +155,11 @@ def load_pair_history(pair: str, # The user forced the refresh of pairs if refresh_pairs: - download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) + _download_pair_history(datadir=datadir, + exchange=exchange, + pair=pair, + timeframe=timeframe, + timerange=timerange) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) @@ -277,11 +277,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, return (data, since_ms) -def download_pair_history(datadir: Path, - exchange: Optional[Exchange], - pair: str, - timeframe: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: +def _download_pair_history(datadir: Path, + exchange: Optional[Exchange], + pair: str, + timeframe: str = '5m', + timerange: Optional[TimeRange] = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -312,11 +312,12 @@ def download_pair_history(datadir: Path, logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given - new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms if since_ms - else + new_data = exchange.get_historic_ohlcv(pair=pair, + timeframe=timeframe, + since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000) + days=-30).float_timestamp) * 1000 + ) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) @@ -334,12 +335,12 @@ def download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], - dl_path: Path, timerange: Optional[TimeRange] = None, + datadir: Path, timerange: Optional[TimeRange] = None, erase=False) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. - Used by freqtrade download-data - :return: Pairs not available + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: @@ -349,23 +350,23 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes continue for timeframe in timeframes: - dl_file = pair_data_filename(dl_path, pair, timeframe) + dl_file = pair_data_filename(datadir, pair, timeframe) if erase and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {timeframe}.') dl_file.unlink() logger.info(f'Downloading pair {pair}, interval {timeframe}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, timeframe=str(timeframe), - timerange=timerange) + _download_pair_history(datadir=datadir, exchange=exchange, + pair=pair, timeframe=str(timeframe), + timerange=timerange) return pairs_not_available -def download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: +def _download_trades_history(datadir: Path, + exchange: Exchange, + pair: str, + timerange: Optional[TimeRange] = None) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. @@ -381,11 +382,11 @@ def download_trades_history(datadir: Path, logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') + # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, since=since if since else int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000, - # until=xxx, from_id=from_id, ) trades.extend(new_trades[1]) @@ -407,9 +408,9 @@ def download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, timerange: TimeRange, erase=False) -> List[str]: """ - Refresh stored trades data. - Used by freqtrade download-data - :return: Pairs not available + Refresh stored trades data for backtesting and hyperopt operations. + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: @@ -425,9 +426,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: dl_file.unlink() logger.info(f'Downloading trades for pair {pair}.') - download_trades_history(datadir=datadir, exchange=exchange, - pair=pair, - timerange=timerange) + _download_trades_history(datadir=datadir, exchange=exchange, + pair=pair, + timerange=timerange) return pairs_not_available diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 230fcf268..9e01c7ea6 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) + datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6d89ab7c5..c304aa97d 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -14,9 +14,9 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import (_load_cached_data_for_updating, + _download_pair_history, + _download_trades_history, convert_trades_to_ohlcv, - download_pair_history, - download_trades_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, refresh_backtest_ohlcv_data, @@ -267,12 +267,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_1.is_file() assert not file2_1.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='1m') assert not exchange._pairs_last_refresh_time assert file1_1.is_file() assert file2_1.is_file() @@ -284,12 +284,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_5.is_file() assert not file2_5.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='5m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() @@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') assert json_dump_mock.call_count == 2 @@ -324,9 +324,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, _backup_file(file1_1) _backup_file(file1_5) - assert not download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') + assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) @@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], - timeframes=["1m", "5m"], dl_path=testdatadir, + timeframes=["1m", "5m"], datadir=testdatadir, timerange=timerange, erase=True ) @@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): timerange = TimeRange.parse_timerange("20190101-20190102") unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], - dl_path=testdatadir, + datadir=testdatadir, timerange=timerange, erase=False ) @@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -646,8 +646,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() - assert download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() @@ -657,8 +657,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)