From a53c4a3ed1ae8dead2b3e2ee462a686281d7d745 Mon Sep 17 00:00:00 2001 From: Mihail Date: Tue, 2 Sep 2025 15:35:20 +0300 Subject: [PATCH 01/11] Fix the truncation of values by merge_ordered in the merge_informative_pair helper. --- freqtrade/strategy/strategy_helper.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 71dcc3cbf..a1959da91 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -92,6 +92,16 @@ def merge_informative_pair( right_on=date_merge, how="left", ) + + # If date_merge of the informative dataframe candle that is above of first date of dataframe, then first raws of + # dataframe are filled with Nans. The code below is to fix it and fulfill first raws with an appropriate values. + if len(dataframe) > 1 and pd.isnull(dataframe.iloc[0]["date_merge"]): + first_valid_idx = dataframe["date_merge"].first_valid_index() + first_valid_date_merge = dataframe.loc[first_valid_idx, "date_merge"] + matching_informative_raws = informative[informative["date_merge"] < first_valid_date_merge] + if not matching_informative_raws.empty: + dataframe.loc[:first_valid_idx - 1] = dataframe.loc[:first_valid_idx - 1].fillna( + matching_informative_raws.iloc[-1]) else: dataframe = pd.merge( dataframe, informative, left_on="date", right_on=date_merge, how="left" From 3fe721a7727c307518a26dfdc974f97e4d43c646 Mon Sep 17 00:00:00 2001 From: Mihail <30621622+mihalt@users.noreply.github.com> Date: Tue, 2 Sep 2025 18:33:23 +0000 Subject: [PATCH 02/11] Fix for merge_informative_pair fix. --- freqtrade/strategy/strategy_helper.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index a1959da91..4a90d6661 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -95,13 +95,16 @@ def merge_informative_pair( # If date_merge of the informative dataframe candle that is above of first date of dataframe, then first raws of # dataframe are filled with Nans. The code below is to fix it and fulfill first raws with an appropriate values. - if len(dataframe) > 1 and pd.isnull(dataframe.iloc[0]["date_merge"]): - first_valid_idx = dataframe["date_merge"].first_valid_index() - first_valid_date_merge = dataframe.loc[first_valid_idx, "date_merge"] - matching_informative_raws = informative[informative["date_merge"] < first_valid_date_merge] + if len(dataframe) > 1 and len(informative) > 0 and pd.isnull(dataframe.iloc[0][date_merge]): + first_valid_idx = dataframe[date_merge].first_valid_index() + first_valid_date_merge = dataframe.loc[first_valid_idx, date_merge] + matching_informative_raws = informative[ + informative[date_merge] < first_valid_date_merge + ] if not matching_informative_raws.empty: - dataframe.loc[:first_valid_idx - 1] = dataframe.loc[:first_valid_idx - 1].fillna( - matching_informative_raws.iloc[-1]) + dataframe.loc[: first_valid_idx - 1] = dataframe.loc[: first_valid_idx - 1].fillna( + matching_informative_raws.iloc[-1] + ) else: dataframe = pd.merge( dataframe, informative, left_on="date", right_on=date_merge, how="left" From b9482f420db5ee73da454bc4dbaeb41631ad002b Mon Sep 17 00:00:00 2001 From: Mihail <30621622+mihalt@users.noreply.github.com> Date: Tue, 2 Sep 2025 19:25:08 +0000 Subject: [PATCH 03/11] Changed test_merge_informative_pair_monthly test to contain the data from previous informative candle on start too. --- tests/strategy/test_strategy_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 605579191..8a2b4083b 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -101,6 +101,7 @@ def test_merge_informative_pair_monthly(): candle1 = result.loc[(result["date"] == "2022-12-31T22:00:00.000Z")] assert candle1.iloc[0]["date"] == pd.Timestamp("2022-12-31T22:00:00.000Z") assert candle1.iloc[0]["date_1M"] == pd.Timestamp("2022-11-01T00:00:00.000Z") + assert candle1.iloc[0]["volume_1M"] candle2 = result.loc[(result["date"] == "2022-12-31T23:00:00.000Z")] assert candle2.iloc[0]["date"] == pd.Timestamp("2022-12-31T23:00:00.000Z") @@ -109,7 +110,8 @@ def test_merge_informative_pair_monthly(): # Candle is empty, as the start-date did fail. candle3 = result.loc[(result["date"] == "2022-11-30T22:00:00.000Z")] assert candle3.iloc[0]["date"] == pd.Timestamp("2022-11-30T22:00:00.000Z") - assert candle3.iloc[0]["date_1M"] is pd.NaT + assert candle3.iloc[0]["date_1M"] + assert candle3.iloc[0]["volume_1M"] # First candle with 1M data merged. candle4 = result.loc[(result["date"] == "2022-11-30T23:00:00.000Z")] From d36fdc1d6992ecc85df3526ed872603781c13134 Mon Sep 17 00:00:00 2001 From: Mihail <30621622+mihalt@users.noreply.github.com> Date: Wed, 3 Sep 2025 12:20:38 +0000 Subject: [PATCH 04/11] Added exact dates to test_merge_informative_pair_monthly. --- tests/strategy/test_strategy_helpers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 8a2b4083b..710dcc559 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -101,7 +101,7 @@ def test_merge_informative_pair_monthly(): candle1 = result.loc[(result["date"] == "2022-12-31T22:00:00.000Z")] assert candle1.iloc[0]["date"] == pd.Timestamp("2022-12-31T22:00:00.000Z") assert candle1.iloc[0]["date_1M"] == pd.Timestamp("2022-11-01T00:00:00.000Z") - assert candle1.iloc[0]["volume_1M"] + assert candle1.iloc[0]["volume_1M"] == np.float64(199.11048557037446) candle2 = result.loc[(result["date"] == "2022-12-31T23:00:00.000Z")] assert candle2.iloc[0]["date"] == pd.Timestamp("2022-12-31T23:00:00.000Z") @@ -110,8 +110,7 @@ def test_merge_informative_pair_monthly(): # Candle is empty, as the start-date did fail. candle3 = result.loc[(result["date"] == "2022-11-30T22:00:00.000Z")] assert candle3.iloc[0]["date"] == pd.Timestamp("2022-11-30T22:00:00.000Z") - assert candle3.iloc[0]["date_1M"] - assert candle3.iloc[0]["volume_1M"] + assert candle3.iloc[0]["volume_1M"] == np.float64(199.2462638356425) # First candle with 1M data merged. candle4 = result.loc[(result["date"] == "2022-11-30T23:00:00.000Z")] From 70d7dcd189ff44cb9dfdba47ea03be13792b04fe Mon Sep 17 00:00:00 2001 From: Mihail <30621622+mihalt@users.noreply.github.com> Date: Wed, 3 Sep 2025 19:55:37 +0000 Subject: [PATCH 05/11] Made test_merge_informative_pair_monthly asserts data dependent. --- tests/strategy/test_strategy_helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 710dcc559..982dd184f 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -101,7 +101,8 @@ def test_merge_informative_pair_monthly(): candle1 = result.loc[(result["date"] == "2022-12-31T22:00:00.000Z")] assert candle1.iloc[0]["date"] == pd.Timestamp("2022-12-31T22:00:00.000Z") assert candle1.iloc[0]["date_1M"] == pd.Timestamp("2022-11-01T00:00:00.000Z") - assert candle1.iloc[0]["volume_1M"] == np.float64(199.11048557037446) + prev_m_vol = informative.loc[informative["date"] == "2022-11-01T00:00:00.000", "volume"] + assert candle1.iloc[0]["volume_1M"] == prev_m_vol.iloc[0] candle2 = result.loc[(result["date"] == "2022-12-31T23:00:00.000Z")] assert candle2.iloc[0]["date"] == pd.Timestamp("2022-12-31T23:00:00.000Z") @@ -110,7 +111,8 @@ def test_merge_informative_pair_monthly(): # Candle is empty, as the start-date did fail. candle3 = result.loc[(result["date"] == "2022-11-30T22:00:00.000Z")] assert candle3.iloc[0]["date"] == pd.Timestamp("2022-11-30T22:00:00.000Z") - assert candle3.iloc[0]["volume_1M"] == np.float64(199.2462638356425) + prev_m_vol = informative.loc[informative["date"] == "2022-10-01 00:00:00+00:00", "volume"] + assert candle3.iloc[0]["volume_1M"] == prev_m_vol.iloc[0] # First candle with 1M data merged. candle4 = result.loc[(result["date"] == "2022-11-30T23:00:00.000Z")] From 0111e97856e08b6e08217db371681991683e4bb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 Sep 2025 06:45:27 +0200 Subject: [PATCH 06/11] test: update test --- tests/strategy/test_strategy_helpers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 982dd184f..e5486f8ae 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -101,8 +101,6 @@ def test_merge_informative_pair_monthly(): candle1 = result.loc[(result["date"] == "2022-12-31T22:00:00.000Z")] assert candle1.iloc[0]["date"] == pd.Timestamp("2022-12-31T22:00:00.000Z") assert candle1.iloc[0]["date_1M"] == pd.Timestamp("2022-11-01T00:00:00.000Z") - prev_m_vol = informative.loc[informative["date"] == "2022-11-01T00:00:00.000", "volume"] - assert candle1.iloc[0]["volume_1M"] == prev_m_vol.iloc[0] candle2 = result.loc[(result["date"] == "2022-12-31T23:00:00.000Z")] assert candle2.iloc[0]["date"] == pd.Timestamp("2022-12-31T23:00:00.000Z") @@ -111,14 +109,20 @@ def test_merge_informative_pair_monthly(): # Candle is empty, as the start-date did fail. candle3 = result.loc[(result["date"] == "2022-11-30T22:00:00.000Z")] assert candle3.iloc[0]["date"] == pd.Timestamp("2022-11-30T22:00:00.000Z") - prev_m_vol = informative.loc[informative["date"] == "2022-10-01 00:00:00+00:00", "volume"] - assert candle3.iloc[0]["volume_1M"] == prev_m_vol.iloc[0] + # Merged on prior month + assert candle3.iloc[0]["date_1M"] == pd.Timestamp("2022-10-01T00:00:00.000Z") # First candle with 1M data merged. candle4 = result.loc[(result["date"] == "2022-11-30T23:00:00.000Z")] assert candle4.iloc[0]["date"] == pd.Timestamp("2022-11-30T23:00:00.000Z") assert candle4.iloc[0]["date_1M"] == pd.Timestamp("2022-11-01T00:00:00.000Z") + # Very first candle in the result dataframe + # Merged the latest informative candle before the start-date + candle5 = result.iloc[0] + assert candle5["date"] == pd.Timestamp("2022-11-28T00:00:00.000Z") + assert candle5["date_1M"] == pd.Timestamp("2022-10-01T00:00:00.000Z") + def test_merge_informative_pair_same(): data = generate_test_data("15m", 40) From 88cc24c5b96fed30cad1d299eeee748253f2614c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 Sep 2025 06:55:15 +0200 Subject: [PATCH 07/11] chore: fix odd code comment --- freqtrade/strategy/strategy_helper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 4a90d6661..db2ba88fa 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -93,9 +93,10 @@ def merge_informative_pair( how="left", ) - # If date_merge of the informative dataframe candle that is above of first date of dataframe, then first raws of - # dataframe are filled with Nans. The code below is to fix it and fulfill first raws with an appropriate values. if len(dataframe) > 1 and len(informative) > 0 and pd.isnull(dataframe.iloc[0][date_merge]): + # If the start dates of the dataframes are not aligned, the first rows will be NaN + # We can fill these with the last available informative candle before the start date + # while still avoiding lookahead bias - as only past data is used. first_valid_idx = dataframe[date_merge].first_valid_index() first_valid_date_merge = dataframe.loc[first_valid_idx, date_merge] matching_informative_raws = informative[ From 747eac04171c7a9ad548a704a14c5a4dacc5de2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 12 Sep 2025 06:58:22 +0200 Subject: [PATCH 08/11] chore: update code to use at which has better performance than chaining iloc and column selection --- freqtrade/strategy/strategy_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index db2ba88fa..34d54f9a4 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -93,12 +93,12 @@ def merge_informative_pair( how="left", ) - if len(dataframe) > 1 and len(informative) > 0 and pd.isnull(dataframe.iloc[0][date_merge]): + if len(dataframe) > 1 and len(informative) > 0 and pd.isnull(dataframe.at[0, date_merge]): # If the start dates of the dataframes are not aligned, the first rows will be NaN # We can fill these with the last available informative candle before the start date # while still avoiding lookahead bias - as only past data is used. first_valid_idx = dataframe[date_merge].first_valid_index() - first_valid_date_merge = dataframe.loc[first_valid_idx, date_merge] + first_valid_date_merge = dataframe.at[first_valid_idx, date_merge] matching_informative_raws = informative[ informative[date_merge] < first_valid_date_merge ] From 0ca846b2c5591c2f89d9c5baa83e101ce464e48b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 Sep 2025 10:34:11 +0200 Subject: [PATCH 09/11] test: update test comment --- tests/strategy/test_strategy_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index e5486f8ae..c3ee01c19 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -34,7 +34,8 @@ def test_merge_informative_pair(): assert "volume_1h" in result.columns assert result["volume"].equals(data["volume"]) - # First 3 rows are empty + # First 3 rows are empty. + # Pre-fillup doesn't happen as there is no prior candlw in the informative dataframe assert result.iloc[0]["date_1h"] is pd.NaT assert result.iloc[1]["date_1h"] is pd.NaT assert result.iloc[2]["date_1h"] is pd.NaT From 876875cff2088421a6628ed6b76927df64e87581 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Oct 2025 09:51:15 +0200 Subject: [PATCH 10/11] test: add test-case for no overlap merges --- tests/strategy/test_strategy_helpers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index c3ee01c19..e855668c6 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -125,6 +125,23 @@ def test_merge_informative_pair_monthly(): assert candle5["date_1M"] == pd.Timestamp("2022-10-01T00:00:00.000Z") +def test_merge_informative_pair_no_overlap(): + # Covers roughly a day + data = generate_test_data("1m", 1440, "2022-11-28") + # Data stops WAY before the main data starts + informative = generate_test_data("1h", 40, "2022-11-01") + + result = merge_informative_pair(data, informative, "1m", "1h", ffill=True) + + assert isinstance(result, pd.DataFrame) + assert len(result) == len(data) + assert "date" in result.columns + assert result["date"].equals(data["date"]) + assert "date_1h" in result.columns + # If there's no overlap, forward filling should not fill anything + assert result["date_1h"].isnull().all() + + def test_merge_informative_pair_same(): data = generate_test_data("15m", 40) informative = generate_test_data("15m", 40) From 93b87696c6afa62e921fd08029a4fc9707a2e1a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Oct 2025 09:51:31 +0200 Subject: [PATCH 11/11] fix: handle case where informative does not overlap with the main data --- freqtrade/strategy/strategy_helper.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 34d54f9a4..1292b65b5 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -98,14 +98,15 @@ def merge_informative_pair( # We can fill these with the last available informative candle before the start date # while still avoiding lookahead bias - as only past data is used. first_valid_idx = dataframe[date_merge].first_valid_index() - first_valid_date_merge = dataframe.at[first_valid_idx, date_merge] - matching_informative_raws = informative[ - informative[date_merge] < first_valid_date_merge - ] - if not matching_informative_raws.empty: - dataframe.loc[: first_valid_idx - 1] = dataframe.loc[: first_valid_idx - 1].fillna( - matching_informative_raws.iloc[-1] - ) + if first_valid_idx: + first_valid_date_merge = dataframe.at[first_valid_idx, date_merge] + matching_informative_raws = informative[ + informative[date_merge] < first_valid_date_merge + ] + if not matching_informative_raws.empty: + dataframe.loc[: first_valid_idx - 1] = dataframe.loc[ + : first_valid_idx - 1 + ].fillna(matching_informative_raws.iloc[-1]) else: dataframe = pd.merge( dataframe, informative, left_on="date", right_on=date_merge, how="left"