diff --git a/docs/backtesting.md b/docs/backtesting.md index 6db573224..837fcfa3b 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -103,12 +103,6 @@ The full timerange specification: - Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` -- Use last 123 tickframes of data: `--timerange=-123` -- Use first 123 tickframes of data: `--timerange=123-` -- Use tickframes from line 123 through 456: `--timerange=123-456` - -!!! warning - Be carefull when using non-date functions - these do not allow you to specify precise dates, so if you updated the test-data it will probably use a different dataset. ## Understand the backtesting result diff --git a/docs/edge.md b/docs/edge.md index d91522770..6374da9e6 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -249,13 +249,10 @@ freqtrade edge --stoplosses=-0.01,-0.1,-0.001 #min,max,step freqtrade edge --timerange=20181110-20181113 ``` -Doing `--timerange=-200` will get the last 200 timeframes from your inputdata. You can also specify specific dates, or a range span indexed by start and stop. +Doing `--timerange=-20190901` will get all available data until September 1st (excluding September 1st 2019). The full timerange specification: -* Use last 123 tickframes of data: `--timerange=-123` -* Use first 123 tickframes of data: `--timerange=123-` -* Use tickframes from line 123 through 456: `--timerange=123-456` * Use tickframes till 2018/01/31: `--timerange=-20180131` * Use tickframes since 2018/01/31: `--timerange=20180131-` * Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index ec6eb75e6..fc759ab6e 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -42,9 +42,10 @@ class TimeRange: (r'^-(\d{10})$', (None, 'date')), (r'^(\d{10})-$', ('date', None)), (r'^(\d{10})-(\d{10})$', ('date', 'date')), - (r'^(-\d+)$', (None, 'line')), - (r'^(\d+)-$', ('line', None)), - (r'^(\d+)-(\d+)$', ('index', 'index'))] + (r'^-(\d{13})$', (None, 'date')), + (r'^(\d{13})-$', ('date', None)), + (r'^(\d{13})-(\d{13})$', ('date', 'date')), + ] for rex, stype in syntax: # Apply the regular expression to text match = re.match(rex, text) @@ -57,6 +58,8 @@ class TimeRange: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: start = arrow.get(starts, 'YYYYMMDD').timestamp + elif len(starts) == 13: + start = int(starts) // 1000 else: start = int(starts) index += 1 @@ -64,6 +67,8 @@ class TimeRange: stops = rvals[index] if stype[1] == 'date' and len(stops) == 8: stop = arrow.get(stops, 'YYYYMMDD').timestamp + elif len(stops) == 13: + stop = int(stops) // 1000 else: stop = int(stops) return TimeRange(stype[0], stype[1], start, stop) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index fd3aee2a9..ed5d80b0e 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -33,20 +33,12 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: start_index = 0 stop_index = len(tickerlist) - if timerange.starttype == 'line': - stop_index = timerange.startts - if timerange.starttype == 'index': - start_index = timerange.startts - elif timerange.starttype == 'date': + if timerange.starttype == 'date': while (start_index < len(tickerlist) and tickerlist[start_index][0] < timerange.startts * 1000): start_index += 1 - if timerange.stoptype == 'line': - start_index = max(len(tickerlist) + timerange.stopts, 0) - if timerange.stoptype == 'index': - stop_index = timerange.stopts - elif timerange.stoptype == 'date': + if timerange.stoptype == 'date': while (stop_index > 0 and tickerlist[stop_index-1][0] > timerange.stopts * 1000): stop_index -= 1 diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 27a8ee02e..4068e00e4 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -53,12 +53,12 @@ def test_load_trades_db(default_conf, fee, mocker): def test_extract_trades_of_period(testdatadir): pair = "UNITTEST/BTC" - timerange = TimeRange(None, 'line', 0, -1000) + # 2018-11-14 06:07:00 + timerange = TimeRange('date', None, 1510639620, 0) data = load_pair_history(pair=pair, ticker_interval='1m', datadir=testdatadir, timerange=timerange) - # timerange = 2017-11-14 06:07 - 2017-11-14 22:58:00 trades = DataFrame( {'pair': [pair, pair, pair, pair], 'profit_percent': [0.0, 0.1, -0.2, -0.5], diff --git a/tests/data/test_history.py b/tests/data/test_history.py index d653428e9..95382768a 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -381,37 +381,6 @@ def test_trim_tickerlist(testdatadir) -> None: ticker_list = json.load(data_file) ticker_list_len = len(ticker_list) - # Test the pattern ^(-\d+)$ - # This pattern uses the latest N elements - timerange = TimeRange(None, 'line', 0, -5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[-1] is ticker[-1] # The last element must be the same - - # Test the pattern ^(\d+)-$ - # This pattern keep X element from the end - timerange = TimeRange('line', None, 5, 0) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is ticker[0] # The first element must be the same - assert ticker_list[-1] is not ticker[-1] # The last element should be different - - # Test the pattern ^(\d+)-(\d+)$ - # This pattern extract a window - timerange = TimeRange('index', 'index', 5, 10) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - # Test the pattern ^(\d{8})-(\d{8})$ # This pattern extract a window between the dates timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) @@ -451,13 +420,6 @@ def test_trim_tickerlist(testdatadir) -> None: assert ticker_list_len == ticker_len - # Test invalid timerange (start after stop) - timerange = TimeRange('index', 'index', 10, 5) - with pytest.raises(ValueError, match=r'The timerange .* is incorrect'): - trim_tickerlist(ticker_list, timerange) - - assert ticker_list_len == ticker_len - # passing empty list timerange = TimeRange(None, None, None, 5) ticker = trim_tickerlist([], timerange) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2b6f63516..998edda8a 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -49,7 +49,7 @@ def trim_dictlist(dict_list, num): def load_data_test(what, testdatadir): - timerange = TimeRange(None, 'line', 0, -101) + timerange = TimeRange.parse_timerange('1510694220-1510700340') pair = history.load_tickerdata_file(testdatadir, ticker_interval='1m', pair='UNITTEST/BTC', timerange=timerange) datalen = len(pair) @@ -342,7 +342,8 @@ def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None: def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) - timerange = TimeRange(None, 'line', 0, -100) + # timerange = TimeRange(None, 'line', 0, -100) + timerange = TimeRange.parse_timerange('1510694220-1510700340') tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} @@ -474,7 +475,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: default_conf['ticker_interval'] = '1m' default_conf['datadir'] = testdatadir default_conf['export'] = None - default_conf['timerange'] = '-100' + default_conf['timerange'] = '-1510694220' backtesting = Backtesting(default_conf) backtesting.start() @@ -522,7 +523,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) pair = 'UNITTEST/BTC' - timerange = TimeRange(None, 'line', 0, -201) + timerange = TimeRange('date', None, 1517227800, 0) data = history.load_data(datadir=testdatadir, ticker_interval='5m', pairs=['UNITTEST/BTC'], timerange=timerange) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) @@ -578,7 +579,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - backtesting = Backtesting(default_conf) # Run a backtesting for an exiting 1min ticker_interval - timerange = TimeRange(None, 'line', 0, -200) + timerange = TimeRange.parse_timerange('1510688220-1510700340') data = history.load_data(datadir=testdatadir, ticker_interval='1m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.tickerdata_to_dataframe(data) @@ -823,7 +824,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): '--datadir', str(testdatadir), 'backtesting', '--ticker-interval', '1m', - '--timerange', '-100', + '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions' ] @@ -833,7 +834,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): exists = [ 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', - 'Parameter --timerange detected: -100 ...', + 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', @@ -869,7 +870,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): '--datadir', str(testdatadir), 'backtesting', '--ticker-interval', '1m', - '--timerange', '-100', + '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', '--strategy-list', @@ -887,7 +888,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): exists = [ 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', - 'Parameter --timerange detected: -100 ...', + 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 094cd41a1..5519b1a34 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -106,7 +106,7 @@ def test_get_signal_handles_exceptions(mocker, default_conf): def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None: strategy = DefaultStrategy(default_conf) - timerange = TimeRange(None, 'line', 0, -100) + timerange = TimeRange.parse_timerange('1510694220-1510700340') tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", fill_missing=True)} diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 6599472fb..4851cbebd 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -5,9 +5,6 @@ from freqtrade.configuration import TimeRange def test_parse_timerange_incorrect() -> None: - assert TimeRange(None, 'line', 0, -200) == TimeRange.parse_timerange('-200') - assert TimeRange('line', None, 200, 0) == TimeRange.parse_timerange('200-') - assert TimeRange('index', 'index', 200, 500) == TimeRange.parse_timerange('200-500') assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-') assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522') @@ -20,9 +17,14 @@ def test_parse_timerange_incorrect() -> None: timerange = TimeRange.parse_timerange('1231006505-1233360000') assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange - # TODO: Find solution for the following case (passing timestamp in ms) timerange = TimeRange.parse_timerange('1231006505000-1233360000000') - assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange + assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + + timerange = TimeRange.parse_timerange('1231006505000-') + assert TimeRange('date', None, 1231006505, 0) == timerange + + timerange = TimeRange.parse_timerange('-1231006505000') + assert TimeRange(None, 'date', 0, 1231006505) == timerange with pytest.raises(Exception, match=r'Incorrect syntax.*'): TimeRange.parse_timerange('-')