From 6cb17caf918d238b2971f7e84f069c44e2106f7f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Oct 2024 20:28:26 +0100 Subject: [PATCH 1/5] tests: Add test showing behavior from #10853 --- tests/optimize/test_backtesting.py | 125 +++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f064247c6..e04ebae0f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1681,6 +1681,131 @@ def test_backtest_multi_pair_detail( assert len(evaluate_result_multi(results["results"], "5m", 1)) == 0 +@pytest.mark.parametrize("use_detail", [True, False]) +@pytest.mark.parametrize("pair", ["ADA/USDT", "LTC/USDT"]) +@pytest.mark.parametrize("tres", [0, 20, 30]) +def test_backtest_multi_pair_detail_simplified( + default_conf_usdt, + fee, + mocker, + tres, + pair, + use_detail, +): + """ + literally the same as test_backtest_multi_pair_detail + but with an "always enter" strategy, exiting after about half of the candle duration. + """ + + def _always_buy(dataframe, metadata): + """ + Buy every xth candle - sell every other xth -2 (hold on to pairs a bit) + """ + dataframe["enter_long"] = 1 + dataframe["enter_short"] = 0 + dataframe["exit_short"] = 0 + return dataframe + + def custom_exit( + trade: Trade, + current_time: datetime, + **kwargs, + ) -> str | bool | None: + # Exit within the same candle. + if (trade.open_date_utc + timedelta(minutes=20)) < current_time: + return "exit after 20 minutes" + + default_conf_usdt.update( + { + "runmode": "backtest", + "stoploss": -1.0, + "minimal_roi": {"0": 100}, + } + ) + + if use_detail: + default_conf_usdt["timeframe_detail"] = "5m" + + mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) + mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float("inf")) + mocker.patch(f"{EXMS}.get_fee", fee) + patch_exchange(mocker) + + raw_candles_5m = generate_test_data("5m", 1000, "2022-01-03 12:00:00+00:00") + raw_candles = ohlcv_fill_up_missing_data(raw_candles_5m, "1h", "dummy") + + pairs = ["ADA/USDT", "DASH/USDT", "ETH/USDT", "LTC/USDT", "NXT/USDT"] + data = {pair: raw_candles for pair in pairs} + detail_data = {pair: raw_candles_5m for pair in pairs} + + # Only use 500 lines to increase performance + data = trim_dictlist(data, -200) + + # Remove data for one pair from the beginning of the data + if tres > 0: + data[pair] = data[pair][tres:].reset_index() + default_conf_usdt["timeframe"] = "1h" + default_conf_usdt["max_open_trades"] = 3 + + backtesting = Backtesting(default_conf_usdt) + vr_spy = mocker.spy(backtesting, "validate_row") + bl_spy = mocker.spy(backtesting, "backtest_loop") + backtesting.detail_data = detail_data + backtesting._set_strategy(backtesting.strategylist[0]) + backtesting.strategy.bot_loop_start = MagicMock() + backtesting.strategy.advise_entry = _always_buy # Override + backtesting.strategy.advise_exit = _always_buy # Override + backtesting.strategy.custom_exit = custom_exit # Override + + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + + backtest_conf = { + "processed": deepcopy(processed), + "start_date": min_date, + "end_date": max_date, + } + + results = backtesting.backtest(**backtest_conf) + + # bot_loop_start is called once per candle. + # assert backtesting.strategy.bot_loop_start.call_count == 83 + # Validated row once per candle and pair + assert vr_spy.call_count == 415 + + if use_detail: + # Backtest loop is called once per candle per pair + # Exact numbers depend on trade state - but should be around 3_800 + assert bl_spy.call_count > 1_350 + # assert bl_spy.call_count < 1_500 + else: + assert bl_spy.call_count < 995 + + # Make sure we have parallel trades + assert len(evaluate_result_multi(results["results"], "1h", 2)) > 0 + # make sure we don't have trades with more than configured max_open_trades + assert len(evaluate_result_multi(results["results"], "1h", 3)) == 0 + + # # Cached data correctly removed amounts + offset = 1 if tres == 0 else 0 + removed_candles = len(data[pair]) - offset + assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, "1h")[0]) == removed_candles + assert ( + len(backtesting.dataprovider.get_analyzed_dataframe("NXT/USDT", "1h")[0]) + == len(data["NXT/USDT"]) - 1 + ) + + backtesting.strategy.max_open_trades = 1 + backtesting.config.update({"max_open_trades": 1}) + backtest_conf = { + "processed": deepcopy(processed), + "start_date": min_date, + "end_date": max_date, + } + results = backtesting.backtest(**backtest_conf) + assert len(evaluate_result_multi(results["results"], "1h", 1)) == 0 + + @pytest.mark.parametrize("use_detail", [True, False]) def test_backtest_multi_pair_long_short_switch( default_conf_usdt, From 56df7ad9fc228e9777e919477e0b9895087db473 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2024 06:57:52 +0100 Subject: [PATCH 2/5] chore: remove unnecessary commented log statement --- freqtrade/optimize/backtesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 03c7bcccc..4b8091f87 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -732,7 +732,6 @@ class Backtesting: trade.close_date = current_time trade.close(order.ft_price, show_msg=False) - # logger.debug(f"{pair} - Backtesting exit {trade}") LocalTrade.close_bt_trade(trade) self.wallets.update() self.run_protections(pair, current_time, trade.trade_direction) From 26a59e2cc54146042e51a816fd95e5fe6a38c4f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2024 07:13:02 +0100 Subject: [PATCH 3/5] fix: Don't allow backtesting to exceed max_open_trades in odd edge-cases closes #10853 --- freqtrade/optimize/backtesting.py | 7 ++++++- freqtrade/persistence/trade_model.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b8091f87..fd4c98b4d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1102,6 +1102,7 @@ class Backtesting: fee_close=self.fee, is_open=True, enter_tag=entry_tag, + timeframe=self.timeframe_min, exchange=self._exchange_name, is_short=is_short, trading_mode=self.trading_mode, @@ -1357,7 +1358,7 @@ class Backtesting: and (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0) and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir) ): - if self.trade_slot_available(LocalTrade.bt_open_open_trade_count): + if self.trade_slot_available(LocalTrade.bt_open_open_trade_count_candle): trade = self._enter_trade(pair, row, trade_dir) if trade: self.wallets.update() @@ -1431,6 +1432,10 @@ class Backtesting: ): if is_first_call: self.check_abort() + # Reset open trade count for this candle + # Critical to avoid exceeding max_open_trades in backtesting + # when timeframe-detail is used and trades close within the opening candle. + LocalTrade.bt_open_open_trade_count_candle = LocalTrade.bt_open_open_trade_count strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)( current_time=current_time ) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index afae5f4c7..f765b9865 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -379,6 +379,7 @@ class LocalTrade: # Copy of trades_open - but indexed by pair bt_trades_open_pp: dict[str, list["LocalTrade"]] = defaultdict(list) bt_open_open_trade_count: int = 0 + bt_open_open_trade_count_candle: int = 0 bt_total_profit: float = 0 realized_profit: float = 0 @@ -747,6 +748,7 @@ class LocalTrade: LocalTrade.bt_trades_open = [] LocalTrade.bt_trades_open_pp = defaultdict(list) LocalTrade.bt_open_open_trade_count = 0 + LocalTrade.bt_open_open_trade_count_candle = 0 LocalTrade.bt_total_profit = 0 def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None: @@ -1442,6 +1444,11 @@ class LocalTrade: LocalTrade.bt_trades_open.remove(trade) LocalTrade.bt_trades_open_pp[trade.pair].remove(trade) LocalTrade.bt_open_open_trade_count -= 1 + if (trade.close_date_utc - trade.open_date_utc) > timedelta(minutes=trade.timeframe): + # Only subtract trades that are open for more than 1 candle + # To avoid exceeding max_open_trades. + # Must be reset at the start of every candle during backesting. + LocalTrade.bt_open_open_trade_count_candle -= 1 LocalTrade.bt_trades.append(trade) LocalTrade.bt_total_profit += trade.close_profit_abs @@ -1451,6 +1458,7 @@ class LocalTrade: LocalTrade.bt_trades_open.append(trade) LocalTrade.bt_trades_open_pp[trade.pair].append(trade) LocalTrade.bt_open_open_trade_count += 1 + LocalTrade.bt_open_open_trade_count_candle += 1 else: LocalTrade.bt_trades.append(trade) @@ -1459,6 +1467,9 @@ class LocalTrade: LocalTrade.bt_trades_open.remove(trade) LocalTrade.bt_trades_open_pp[trade.pair].remove(trade) LocalTrade.bt_open_open_trade_count -= 1 + # TODO: The below may have odd behavior in case of canceled entries + # It might need to be removed so the trade "counts" as open for this candle. + LocalTrade.bt_open_open_trade_count_candle -= 1 @staticmethod def get_open_trades() -> list[Any]: From e35f5c9fded4f65e6f27c4fa57f6c2fe507d9728 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2024 07:13:36 +0100 Subject: [PATCH 4/5] tests: update test to account for newly added prop --- tests/persistence/test_persistence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 05160c74a..b92791ca5 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2144,6 +2144,7 @@ def test_Trade_object_idem(): "bt_trades_open", "bt_trades_open_pp", "bt_open_open_trade_count", + "bt_open_open_trade_count_candle", "bt_total_profit", "from_json", ) From 592848ad0316c531eaefedac1eaa0a8fcb3264be Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Oct 2024 19:27:03 +0100 Subject: [PATCH 5/5] test: re-add excluded assert --- tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index e04ebae0f..416ffa262 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1776,8 +1776,8 @@ def test_backtest_multi_pair_detail_simplified( if use_detail: # Backtest loop is called once per candle per pair # Exact numbers depend on trade state - but should be around 3_800 - assert bl_spy.call_count > 1_350 - # assert bl_spy.call_count < 1_500 + assert bl_spy.call_count > 3_350 + assert bl_spy.call_count < 3_800 else: assert bl_spy.call_count < 995