From 11dce9128193184623d9a4896d503c351ef37faf Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:02 +0300 Subject: [PATCH 1/3] data/history minor cleanup --- freqtrade/data/history.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 86d3c3071..27e68b533 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -63,12 +63,8 @@ def load_tickerdata_file( Load a pair from file, either .json.gz or .json :return tickerlist or None if unsuccesful """ - path = make_testdata_path(datadir) - pair_s = pair.replace('/', '_') - file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - - pairdata = misc.file_load_json(file) - + filename = pair_data_filename(datadir, pair, ticker_interval) + pairdata = misc.file_load_json(filename) if not pairdata: return None @@ -142,11 +138,18 @@ def load_data(datadir: Optional[Path], return result -def make_testdata_path(datadir: Optional[Path]) -> Path: +def make_datadir_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() +def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: + path = make_datadir_path(datadir) + pair_s = pair.replace("/", "_") + filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') + return filename + + def load_cached_data_for_updating(filename: Path, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: @@ -209,9 +212,7 @@ def download_pair_history(datadir: Optional[Path], ) try: - path = make_testdata_path(datadir) - filepair = pair.replace("/", "_") - filename = path.joinpath(f'{filepair}-{ticker_interval}.json') + filename = pair_data_filename(datadir, pair, ticker_interval) logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' @@ -236,8 +237,9 @@ def download_pair_history(datadir: Optional[Path], misc.file_dump_json(filename, data) return True - except Exception: + except Exception as e: logger.error( - f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.' + f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}. ' + f'Error: {e}' ) return False From 7cb753754b039898c3cfae2dd8049068cda6b4a2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 21 May 2019 20:49:19 +0300 Subject: [PATCH 2/3] tests adjusted --- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 7 ++++--- freqtrade/tests/test_misc.py | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index dd7cbe0d9..4c0426b93 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_testdata_path +from freqtrade.data.history import make_datadir_path def test_load_backtest_data(): - filename = make_testdata_path(None) / "backtest-result_test.json" + filename = make_datadir_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 15442f577..a37b42351 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_testdata_path, + make_datadir_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) def test_load_cached_data_for_updating(mocker) -> None: @@ -321,7 +321,8 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def _clean_test_file(file1_1) _clean_test_file(file1_5) assert log_has( - 'Failed to download history data for pair: "MEME/BTC", interval: 1m.', + 'Failed to download history data for pair: "MEME/BTC", interval: 1m. ' + 'Error: File Error', caplog.record_tuples ) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 2da6b8718..c7bcf7edf 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, file_dump_json, file_load_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file, make_testdata_path +from freqtrade.data.history import load_tickerdata_file, pair_data_filename from freqtrade.strategy.default_strategy import DefaultStrategy @@ -60,13 +60,13 @@ def test_file_dump_json(mocker) -> None: def test_file_load_json(mocker) -> None: # 7m .json does not exist - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '7m')) assert not ret # 1m json exists (but no .gz exists) - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '1m')) assert ret # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + ret = file_load_json(pair_data_filename(None, 'UNITTEST/BTC', '8m')) assert ret From 98eeec31451c844bbdabc0a8c8dacd2474122596 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 22 May 2019 14:04:58 +0300 Subject: [PATCH 3/3] renaming of make_testdata_path reverted --- freqtrade/data/history.py | 4 ++-- freqtrade/tests/data/test_btanalysis.py | 4 ++-- freqtrade/tests/data/test_history.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 27e68b533..3bec63926 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -138,13 +138,13 @@ def load_data(datadir: Optional[Path], return result -def make_datadir_path(datadir: Optional[Path]) -> Path: +def make_testdata_path(datadir: Optional[Path]) -> Path: """Return the path where testdata files are stored""" return datadir or (Path(__file__).parent.parent / "tests" / "testdata").resolve() def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) -> Path: - path = make_datadir_path(datadir) + path = make_testdata_path(datadir) pair_s = pair.replace("/", "_") filename = path.joinpath(f'{pair_s}-{ticker_interval}.json') return filename diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 4c0426b93..dd7cbe0d9 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -2,12 +2,12 @@ import pytest from pandas import DataFrame from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data -from freqtrade.data.history import make_datadir_path +from freqtrade.data.history import make_testdata_path def test_load_backtest_data(): - filename = make_datadir_path(None) / "backtest-result_test.json" + filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a37b42351..0d4210d3a 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -16,7 +16,7 @@ from freqtrade.data import history from freqtrade.data.history import (download_pair_history, load_cached_data_for_updating, load_tickerdata_file, - make_datadir_path, + make_testdata_path, trim_tickerlist) from freqtrade.misc import file_dump_json from freqtrade.tests.conftest import get_patched_exchange, log_has @@ -136,7 +136,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau def test_testdata_path() -> None: - assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_datadir_path(None)) + assert str(Path('freqtrade') / 'tests' / 'testdata') in str(make_testdata_path(None)) def test_load_cached_data_for_updating(mocker) -> None: