diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index db7ef66fc..8abcc6747 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -149,7 +149,14 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: return data -def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]): +def load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]): + """ + Load one strategy from multi-strategy result + and merge it with results + :param strategy_name: Name of the strategy contained in the result + :param filename: Backtest-result-filename to load + :param results: dict to merge the result to. + """ bt_data = load_backtest_stats(filename) for k in ('metadata', 'strategy'): results[k][strategy_name] = bt_data[k][strategy_name] @@ -160,6 +167,30 @@ def _load_and_merge_backtest_result(strategy_name: str, filename: Path, results: break +def _get_backtest_files(dirname: Path) -> List[Path]: + return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json')))) + + +def get_backtest_resultlist(dirname: Path): + """ + Get list of backtest results read from metadata files + """ + results = [] + for filename in _get_backtest_files(dirname): + metadata = load_backtest_metadata(filename) + if not metadata: + continue + for s, v in metadata.items(): + results.append({ + 'filename': filename.name, + 'strategy': s, + 'run_id': v['run_id'], + 'backtest_start_time': v['backtest_start_time'], + + }) + return results + + def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str], min_backtest_date: datetime = None) -> Dict[str, Any]: """ @@ -179,7 +210,7 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s } # Weird glob expression here avoids including .meta.json files. - for filename in reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))): + for filename in _get_backtest_files(dirname): metadata = load_backtest_metadata(filename) if not metadata: # Files are sorted from newest to oldest. When file without metadata is encountered it @@ -202,7 +233,7 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s if strategy_metadata['run_id'] == run_id: del run_ids[strategy_name] - _load_and_merge_backtest_result(strategy_name, filename, results) + load_and_merge_backtest_result(strategy_name, filename, results) if len(run_ids) == 0: break diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 757ed8aac..a902ea984 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -1,13 +1,16 @@ import asyncio import logging from copy import deepcopy +from typing import Any, Dict, List from fastapi import APIRouter, BackgroundTasks, Depends from freqtrade.configuration.config_validation import validate_config_consistency +from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result from freqtrade.enums import BacktestState from freqtrade.exceptions import DependencyException -from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse +from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest, + BacktestResponse) from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode from freqtrade.rpc.api_server.webserver import ApiServer from freqtrade.rpc.rpc import RPCException @@ -200,3 +203,30 @@ def api_backtest_abort(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest ended", } + + +@router.get('/backtest/history', response_model=List[BacktestHistoryEntry], tags=['webserver', 'backtest']) +def api_backtest_history(config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): + # Get backtest result history, read from metadata files + return get_backtest_resultlist(config['user_data_dir'] / 'backtest_results') + + +@router.get('/backtest/history/result', response_model=BacktestResponse, tags=['webserver', 'backtest']) +def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): + # Get backtest result history, read from metadata files + fn = config['user_data_dir'] / 'backtest_results' / filename + results: Dict[str, Any] = { + 'metadata': {}, + 'strategy': {}, + 'strategy_comparison': [], + } + + load_and_merge_backtest_result(strategy, fn, results) + return { + "status": "ended", + "running": False, + "step": "", + "progress": 1, + "status_msg": "Historic result", + "backtest_result": results, + } diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index ae797edad..a9135cce2 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -421,6 +421,13 @@ class BacktestResponse(BaseModel): backtest_result: Optional[Dict[str, Any]] +class BacktestHistoryEntry(BaseModel): + filename: str + strategy: str + run_id: str + backtest_start_time: int + + class SysInfo(BaseModel): cpu_pct: List[float] ram_pct: float diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d96154824..5021c99f9 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -35,7 +35,8 @@ logger = logging.getLogger(__name__) # 1.13: forcebuy supports stake_amount # versions 2.xx -> futures/short branch # 2.14: Add entry/exit orders to trade response -API_VERSION = 2.14 +# 2.15: Add backtest history endpoints +API_VERSION = 2.15 # Public API, requires no auth. router_public = APIRouter() diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 22869638b..1431bd22a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1429,7 +1429,7 @@ def test_backtesting_show(mocker, testdatadir, capsys): args = [ "backtesting-show", "--export-filename", - f"{testdatadir / 'backtest-result_new.json'}", + f"{testdatadir / 'backtest_results/backtest-result_new.json'}", "--show-pair-list" ] pargs = get_args(args) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index f4275edd9..2b53e4900 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -27,18 +27,19 @@ def test_get_latest_backtest_filename(testdatadir, mocker): with pytest.raises(ValueError, match=r"Directory .* does not seem to contain .*"): - get_latest_backtest_filename(testdatadir.parent) + get_latest_backtest_filename(testdatadir) - res = get_latest_backtest_filename(testdatadir) + testdir_bt = testdatadir / "backtest_results" + res = get_latest_backtest_filename(testdir_bt) assert res == 'backtest-result_new.json' - res = get_latest_backtest_filename(str(testdatadir)) + res = get_latest_backtest_filename(str(testdir_bt)) assert res == 'backtest-result_new.json' mocker.patch("freqtrade.data.btanalysis.json_load", return_value={}) with pytest.raises(ValueError, match=r"Invalid '.last_result.json' format."): - get_latest_backtest_filename(testdatadir) + get_latest_backtest_filename(testdir_bt) def test_get_latest_hyperopt_file(testdatadir): @@ -81,7 +82,7 @@ def test_load_backtest_data_old_format(testdatadir, mocker): def test_load_backtest_data_new_format(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) @@ -92,19 +93,19 @@ def test_load_backtest_data_new_format(testdatadir): assert bt_data.equals(bt_data2) # Test loading from folder (must yield same result) - bt_data3 = load_backtest_data(testdatadir) + bt_data3 = load_backtest_data(testdatadir / "backtest_results") assert bt_data.equals(bt_data3) with pytest.raises(ValueError, match=r"File .* does not exist\."): load_backtest_data(str("filename") + "nofile") with pytest.raises(ValueError, match=r"Unknown dataformat."): - load_backtest_data(testdatadir / LAST_BT_RESULT_FN) + load_backtest_data(testdatadir / "backtest_results" / LAST_BT_RESULT_FN) def test_load_backtest_data_multi(testdatadir): - filename = testdatadir / "backtest-result_multistrat.json" + filename = testdatadir / "backtest_results/backtest-result_multistrat.json" for strategy in ('StrategyTestV2', 'TestStrategy'): bt_data = load_backtest_data(filename, strategy=strategy) assert isinstance(bt_data, DataFrame) @@ -182,7 +183,7 @@ def test_extract_trades_of_period(testdatadir): def test_analyze_trade_parallelism(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) res = analyze_trade_parallelism(bt_data, "5m") @@ -256,7 +257,7 @@ def test_combine_dataframes_with_mean_no_data(testdatadir): def test_create_cum_profit(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") @@ -272,7 +273,7 @@ def test_create_cum_profit(testdatadir): def test_create_cum_profit1(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) # Move close-time to "off" the candle, to make sure the logic still works bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20) @@ -294,7 +295,7 @@ def test_create_cum_profit1(testdatadir): def test_calculate_max_drawdown(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) _, hdate, lowdate, hval, lval, drawdown = calculate_max_drawdown( bt_data, value_col="profit_abs") @@ -318,7 +319,7 @@ def test_calculate_max_drawdown(testdatadir): def test_calculate_csum(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) csum_min, csum_max = calculate_csum(bt_data) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d40fc1e2f..05c0bf575 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -228,7 +228,7 @@ def test_generate_pair_metrics(): def test_generate_daily_stats(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) res = generate_daily_stats(bt_data) assert isinstance(res, dict) @@ -248,7 +248,7 @@ def test_generate_daily_stats(testdatadir): def test_generate_trading_stats(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) res = generate_trading_stats(bt_data) assert isinstance(res, dict) @@ -332,7 +332,7 @@ def test_generate_sell_reason_stats(): def test_text_table_strategy(testdatadir): - filename = testdatadir / "backtest-result_multistrat.json" + filename = testdatadir / "backtest_results/backtest-result_multistrat.json" bt_res_data = load_backtest_stats(filename) bt_res_data_comparison = bt_res_data.pop('strategy_comparison') @@ -364,7 +364,7 @@ def test_generate_edge_table(): def test_generate_periodic_breakdown_stats(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename).to_dict(orient='records') res = generate_periodic_breakdown_stats(bt_data, 'day') @@ -392,7 +392,7 @@ def test__get_resample_from_period(): def test_show_sorted_pairlist(testdatadir, default_conf, capsys): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_stats(filename) default_conf['backtest_show_pair_list'] = True diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 54bf07dc2..af8361571 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1581,6 +1581,38 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert result['status_msg'] == 'Backtest reset' +def test_api_backtest_history(botclient, mocker, testdatadir): + ftbot, client = botclient + mocker.patch('freqtrade.data.btanalysis._get_backtest_files', + return_value=[ + testdatadir / 'backtest_results/backtest-result_multistrat.json', + testdatadir / 'backtest_results/backtest-result_new.json' + ]) + + rc = client_get(client, f"{BASE_URI}/backtest/history") + assert_response(rc, 502) + ftbot.config['user_data_dir'] = testdatadir + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/backtest/history") + assert_response(rc) + result = rc.json() + assert len(result) == 3 + fn = result[0]['filename'] + assert fn == "backtest-result_multistrat.json" + strategy = result[0]['strategy'] + rc = client_get(client, f"{BASE_URI}/backtest/history/result?filename={fn}&strategy={strategy}") + assert_response(rc) + result2 = rc.json() + assert result2 + assert result2['status'] == 'ended' + assert not result2['running'] + assert result2['progress'] == 1 + # Only one strategy loaded - even though we use multiresult + assert len(result2['backtest_result']['strategy']) == 1 + assert result2['backtest_result']['strategy'][strategy] + + def test_health(botclient): ftbot, client = botclient diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 940639465..97f367608 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -157,7 +157,7 @@ def test_plot_trades(testdatadir, caplog): assert fig == fig1 assert log_has("No trades found.", caplog) pair = "ADA/BTC" - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" trades = load_backtest_data(filename) trades = trades.loc[trades['pair'] == pair] @@ -298,7 +298,7 @@ def test_generate_plot_file(mocker, caplog): def test_add_profit(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") @@ -318,7 +318,7 @@ def test_add_profit(testdatadir): def test_generate_profit_graph(testdatadir): - filename = testdatadir / "backtest-result_new.json" + filename = testdatadir / "backtest_results/backtest-result_new.json" trades = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") pairs = ["TRX/BTC", "XLM/BTC"] @@ -456,7 +456,7 @@ def test_plot_profit(default_conf, mocker, testdatadir): match=r"No trades found, cannot generate Profit-plot.*"): plot_profit(default_conf) - default_conf['exportfilename'] = testdatadir / "backtest-result_new.json" + default_conf['exportfilename'] = testdatadir / "backtest_results/backtest-result_new.json" plot_profit(default_conf) diff --git a/tests/testdata/.last_result.json b/tests/testdata/backtest_results/.last_result.json similarity index 100% rename from tests/testdata/.last_result.json rename to tests/testdata/backtest_results/.last_result.json diff --git a/tests/testdata/backtest-result_multistrat.json b/tests/testdata/backtest_results/backtest-result_multistrat.json similarity index 100% rename from tests/testdata/backtest-result_multistrat.json rename to tests/testdata/backtest_results/backtest-result_multistrat.json diff --git a/tests/testdata/backtest_results/backtest-result_multistrat.meta.json b/tests/testdata/backtest_results/backtest-result_multistrat.meta.json new file mode 100644 index 000000000..906edcece --- /dev/null +++ b/tests/testdata/backtest_results/backtest-result_multistrat.meta.json @@ -0,0 +1,10 @@ +{ + "StrategyTestV2": { + "run_id": "430d0271075ef327edbb23088f4db4ebe51a3dbf", + "backtest_start_time": 1648904006 + }, + "TestStrategy": { + "run_id": "110d0271075ef327edbb23085102b4ebe51a3d55", + "backtest_start_time": 1648904006 + } +} diff --git a/tests/testdata/backtest-result_new.json b/tests/testdata/backtest_results/backtest-result_new.json similarity index 100% rename from tests/testdata/backtest-result_new.json rename to tests/testdata/backtest_results/backtest-result_new.json diff --git a/tests/testdata/backtest_results/backtest-result_new.meta.json b/tests/testdata/backtest_results/backtest-result_new.meta.json new file mode 100644 index 000000000..57ecdb19d --- /dev/null +++ b/tests/testdata/backtest_results/backtest-result_new.meta.json @@ -0,0 +1,6 @@ +{ + "StrategyTestV3": { + "run_id": "430d0271075ef327edbb23088f4db4ebe51a3dbf", + "backtest_start_time": 1648904006 + } +}