From 628963c20704ca7d47f4d2ec79cfaaf048fc8c82 Mon Sep 17 00:00:00 2001
From: robcaulk <rob.caulk@gmail.com>
Date: Tue, 12 Sep 2023 12:19:12 +0200
Subject: [PATCH] chore: fix bug associated with leaving FreqAI offline for
 more than 1 candle.

---
 freqtrade/freqai/data_drawer.py              | 50 ++++++++++++++++----
 freqtrade/freqai/freqai_interface.py         |  2 +-
 freqtrade/templates/FreqaiExampleStrategy.py |  2 +-
 3 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py
index b6ded83b1..49c673adc 100644
--- a/freqtrade/freqai/data_drawer.py
+++ b/freqtrade/freqai/data_drawer.py
@@ -263,23 +263,55 @@ class FreqaiDataDrawer:
             self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
             return
 
-    def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None:
+    def set_initial_return_values(self, pair: str,
+                                  pred_df: DataFrame,
+                                  dataframe: DataFrame
+                                  ) -> None:
         """
         Set the initial return values to the historical predictions dataframe. This avoids needing
         to repredict on historical candles, and also stores historical predictions despite
         retrainings (so stored predictions are true predictions, not just inferencing on trained
-        data)
+        data).
+
+        We also aim to keep the date from historical predictions so that the FreqUI displays
+        zeros during any downtime (between FreqAI reloads).
         """
 
-        hist_df = self.historic_predictions
-        len_diff = len(hist_df[pair].index) - len(pred_df.index)
-        if len_diff < 0:
-            df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
-                                  ignore_index=True, keys=hist_df[pair].keys())
+        new_pred = pred_df.copy()
+        # set new_pred values to nans (we want to signal to user that there was nothing
+        # historically made during downtime. The newest pred will get appeneded later in
+        # append_model_predictions)
+        new_pred.iloc[:, :] = np.nan
+        new_pred["date"] = dataframe["date"]
+
+        hist_preds = self.historic_predictions[pair].copy()
+        # rename date_pred column to date so that we can merge on date
+        hist_preds = hist_preds.rename(columns={"date_pred": "date"})
+
+        # find the closest common date between new_pred and historic predictions
+        # and cut off the new_pred dataframe at that date
+        common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner")
+        if len(common_dates.index) > 0:
+            new_pred = new_pred.iloc[len(common_dates):]
         else:
-            df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
+            logger.error("No common dates found between new predictions and historic predictions. "
+                         "You likely left your FreqAI instance offline for more than "
+                         f"{len(dataframe.index)} candles.")
+
+        df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys())
+
+        # remove last row because we will append that later in append_model_predictions()
+        df_concat = df_concat.iloc[:-1]
+        # any missing values will get zeroed out so users can see the exact
+        # downtime in FreqUI
         df_concat = df_concat.fillna(0)
-        self.model_return_values[pair] = df_concat
+
+        # rename date column back to date_pred
+        df_concat = df_concat.rename(columns={"date": "date_pred"})
+
+        self.historic_predictions[pair] = df_concat
+
+        self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True)
 
     def append_model_predictions(self, pair: str, predictions: DataFrame,
                                  do_preds: NDArray[np.int_],
diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py
index efae6d060..f64a3b8f0 100644
--- a/freqtrade/freqai/freqai_interface.py
+++ b/freqtrade/freqai/freqai_interface.py
@@ -453,7 +453,7 @@ class IFreqaiModel(ABC):
             pred_df, do_preds = self.predict(dataframe, dk)
             if pair not in self.dd.historic_predictions:
                 self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
-            self.dd.set_initial_return_values(pair, pred_df)
+            self.dd.set_initial_return_values(pair, pred_df, dataframe)
 
             dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
             return
diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py
index 084cf2e89..e64570b9e 100644
--- a/freqtrade/templates/FreqaiExampleStrategy.py
+++ b/freqtrade/templates/FreqaiExampleStrategy.py
@@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy):
     plot_config = {
         "main_plot": {},
         "subplots": {
-            "&-s_close": {"prediction": {"color": "blue"}},
+            "&-s_close": {"&-s_close": {"color": "blue"}},
             "do_predict": {
                 "do_predict": {"color": "brown"},
             },