From 10917a280a2a72944e3425bc18b4ff2d04570331 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Tue, 31 May 2022 12:26:07 +0300
Subject: [PATCH 01/76] Add initial structure and wrapping.

---
 freqtrade/optimize/backtesting.py            |   5 +-
 freqtrade/persistence/__init__.py            |   1 +
 freqtrade/persistence/keyvalue.py            |  57 ++++++++++
 freqtrade/persistence/keyvalue_middleware.py | 108 +++++++++++++++++++
 freqtrade/persistence/models.py              |   2 +
 freqtrade/persistence/trade_model.py         |  19 ++++
 6 files changed, 191 insertions(+), 1 deletion(-)
 create mode 100644 freqtrade/persistence/keyvalue.py
 create mode 100644 freqtrade/persistence/keyvalue_middleware.py

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 4e604898f..c552d8790 100755
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_signal_candles,
                                                  store_backtest_stats)
-from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
+from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -151,6 +151,7 @@ class Backtesting:
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True
+        KeyValues.use_db = True  # ???
 
     def init_backtest_detail(self):
         # Load detail timeframe if specified
@@ -294,6 +295,8 @@ class Backtesting:
         Trade.use_db = False
         PairLocks.reset_locks()
         Trade.reset_trades()
+        KeyValues.use_db = False
+        KeyValues.reset_keyvalues()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0
diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py
index ab6e2f6a5..0158f588c 100644
--- a/freqtrade/persistence/__init__.py
+++ b/freqtrade/persistence/__init__.py
@@ -1,5 +1,6 @@
 # flake8: noqa: F401
 
+from freqtrade.persistence.keyvalue_middleware import KeyValues
 from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade
diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py
new file mode 100644
index 000000000..60fa903a1
--- /dev/null
+++ b/freqtrade/persistence/keyvalue.py
@@ -0,0 +1,57 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Query, relationship
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence.base import _DECL_BASE
+
+
+class KeyValue(_DECL_BASE):
+    """
+    KeyValue database model
+    Keeps records of metadata as key/value store
+    for trades or global persistant values
+    One to many relationship with Trades:
+      - One trade can have many metadata entries
+      - One metadata entry can only be associated with one Trade
+    """
+    __tablename__ = 'keyvalue'
+    # Uniqueness should be ensured over pair, order_id
+    # its likely that order_id is unique per Pair on some exchanges.
+    __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),)
+
+    id = Column(Integer, primary_key=True)
+    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0)
+
+    trade = relationship("Trade", back_populates="keyvalues")
+
+    kv_key = Column(String(255), nullable=False)
+    kv_type = Column(String(25), nullable=False)
+    kv_value = Column(Text, nullable=False)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=True)
+
+    def __repr__(self):
+        create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.created_at is not None else None)
+        update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
+                       if self.updated_at is not None else None)
+        return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ',
+                f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ',
+                f'updated={update_time})')
+
+    @staticmethod
+    def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
+        """
+        Get all keyvalues, if  trade_id is not specified
+        return will be for generic values not tied to a trade
+        :param trade_id: id of the Trade
+        """
+        key = key if key is not None else "%"
+
+        filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0,
+                   KeyValue.kv_key.ilike(key)]
+
+        return KeyValue.query.filter(*filters)
diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py
new file mode 100644
index 000000000..24c74610a
--- /dev/null
+++ b/freqtrade/persistence/keyvalue_middleware.py
@@ -0,0 +1,108 @@
+import json
+import logging
+from datetime import datetime
+from typing import Any, List, Optional
+
+from freqtrade.persistence.keyvalue import KeyValue
+
+
+logger = logging.getLogger(__name__)
+
+
+class KeyValues():
+    """
+    KeyValues middleware class
+    Abstracts the database layer away so it becomes optional - which will be necessary to support
+    backtesting and hyperopt in the future.
+    """
+
+    use_db = True
+    kvals: List[KeyValue] = []
+    unserialized_types = ['bool', 'float', 'int', 'str']
+
+    @staticmethod
+    def reset_keyvalues() -> None:
+        """
+        Resets all key-value pairs. Only active for backtesting mode.
+        """
+        if not KeyValues.use_db:
+            KeyValues.kvals = []
+
+    @staticmethod
+    def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]:
+        if trade_id is None:
+            trade_id = 0
+
+        if KeyValues.use_db:
+            filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all()
+            for index, kval in enumerate(filtered_kvals):
+                if kval.kv_type not in KeyValues.unserialized_types:
+                    kval.kv_value = json.loads(kval.kv_value)
+                filtered_kvals[index] = kval
+            return filtered_kvals
+        else:
+            filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)]
+            if key is not None:
+                filtered_kvals = [
+                    kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())]
+            return filtered_kvals
+
+    @staticmethod
+    def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None:
+
+        logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}")
+        value_type = type(value).__name__
+        value_db = None
+
+        if value_type not in KeyValues.unserialized_types:
+            try:
+                value_db = json.dumps(value)
+            except TypeError as e:
+                logger.warning(f"could not serialize {key} value due to {e}")
+        else:
+            value_db = str(value)
+
+        if trade_id is None:
+            trade_id = 0
+
+        kvals = KeyValues.get_kval(key=key, trade_id=trade_id)
+        if kvals:
+            kv = kvals[0]
+            kv.kv_value = value
+            kv.updated_at = datetime.utcnow()
+        else:
+            kv = KeyValue(
+                ft_trade_id=trade_id,
+                kv_key=key,
+                kv_type=value_type,
+                kv_value=value,
+                created_at=datetime.utcnow()
+            )
+
+        if KeyValues.use_db and value_db is not None:
+            kv.kv_value = value_db
+            KeyValue.query.session.add(kv)
+            KeyValue.query.session.commit()
+        elif not KeyValues.use_db:
+            kv_index = -1
+            for index, kval in enumerate(KeyValues.kvals):
+                if kval.ft_trade_id == trade_id and kval.kv_key == key:
+                    kv_index = index
+                    break
+
+            if kv_index >= 0:
+                kval.kv_type = value_type
+                kval.value = value
+                kval.updated_at = datetime.utcnow()
+
+                KeyValues.kvals[kv_index] = kval
+            else:
+                KeyValues.kvals.append(kv)
+
+    @staticmethod
+    def get_all_kvals() -> List[KeyValue]:
+
+        if KeyValues.use_db:
+            return KeyValue.query.all()
+        else:
+            return KeyValues.kvals
diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index c31e50892..6a279ab5c 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -10,6 +10,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.keyvalue import KeyValue
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -59,6 +60,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
+    KeyValue.query = Trade._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 358e776e3..b097f6574 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -15,6 +15,8 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.keyvalue import KeyValue
+from freqtrade.persistence.keyvalue_middleware import KeyValues
 
 
 logger = logging.getLogger(__name__)
@@ -206,6 +208,7 @@ class LocalTrade():
     id: int = 0
 
     orders: List[Order] = []
+    keyvalues: List[KeyValue] = []
 
     exchange: str = ''
     pair: str = ''
@@ -870,6 +873,12 @@ class LocalTrade():
                 (o.filled or 0) > 0 and
                 o.status in NON_OPEN_EXCHANGE_STATES]
 
+    def set_kval(self, key: str, value: Any) -> None:
+        KeyValues.set_kval(key=key, value=value, trade_id=self.id)
+
+    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
+        return KeyValues.get_kval(key=key, trade_id=self.id)
+
     @property
     def nr_of_successful_entries(self) -> int:
         """
@@ -1000,6 +1009,7 @@ class Trade(_DECL_BASE, LocalTrade):
     id = Column(Integer, primary_key=True)
 
     orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined")
+    keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan")
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
@@ -1070,6 +1080,9 @@ class Trade(_DECL_BASE, LocalTrade):
         for order in self.orders:
             Order.query.session.delete(order)
 
+        for kval in self.keyvalues:
+            KeyValue.query.session.delete(kval)
+
         Trade.query.session.delete(self)
         Trade.commit()
 
@@ -1345,3 +1358,9 @@ class Trade(_DECL_BASE, LocalTrade):
             .group_by(Trade.pair) \
             .order_by(desc('profit_sum')).first()
         return best_pair
+
+    def set_kval(self, key: str, value: Any) -> None:
+        super().set_kval(key=key, value=value)
+
+    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
+        return super().get_kval(key=key)

From 096e98a68c1e6025da4ffc688928a6e6d27cb20f Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Tue, 31 May 2022 16:16:57 +0300
Subject: [PATCH 02/76] Remove stray debug message.

---
 freqtrade/persistence/keyvalue_middleware.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py
index 24c74610a..8248143ce 100644
--- a/freqtrade/persistence/keyvalue_middleware.py
+++ b/freqtrade/persistence/keyvalue_middleware.py
@@ -50,7 +50,6 @@ class KeyValues():
     @staticmethod
     def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None:
 
-        logger.warning(f"[set_kval] key: {key} trade_id: {trade_id} value: {value}")
         value_type = type(value).__name__
         value_db = None
 

From de01aaf290a965e071b083ae28ce1db31d3f97bf Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Tue, 31 May 2022 16:17:31 +0300
Subject: [PATCH 03/76] Add documentation details.

---
 docs/strategy-advanced.md | 70 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 374c675a2..45961c59d 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -11,7 +11,7 @@ If you're just getting started, please be familiar with the methods described in
 !!! Tip
     You can get a strategy template containing all below methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced`
 
-## Storing information
+## Storing information (Non-Persistent)
 
 Storing information can be accomplished by creating a new dictionary within the strategy class.
 
@@ -40,6 +40,74 @@ class AwesomeStrategy(IStrategy):
 !!! Note
     If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
+## Storing information (Persistent)
+
+Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade.
+Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kval(key='my_key')`.
+Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle.
+For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string.
+
+```python
+from freqtrade.persistence import Trade
+from datetime import timedelta
+
+class AwesomeStrategy(IStrategy):
+
+    def bot_loop_start(self, **kwargs) -> None:
+        for trade in Trade.get_open_order_trades():
+            fills = trade.select_filled_orders(trade.entry_side)
+            if trade.pair == 'ETH/USDT':
+                trade_entry_type = trade.get_kval(key='entry_type')
+                if trade_entry_type is None:
+                    trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
+                elif fills > 1:
+                    trade_entry_type = 'buy_up'
+                trade.set_kval(key='entry_type', value=trade_entry_type)
+        return super().bot_loop_start(**kwargs)
+
+    def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str,
+                           current_time: datetime, proposed_rate: float, current_order_rate: float,
+                           entry_tag: Optional[str], side: str, **kwargs) -> float:
+        # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair.
+        if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc and order.filled == 0.0:
+            dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
+            current_candle = dataframe.iloc[-1].squeeze()
+            # store information about entry adjustment
+            existing_count = trade.get_kval(key='num_entry_adjustments')
+            if not existing_count:
+                existing_count = 1
+            else:
+                existing_count += 1
+            trade.set_kval(key='num_entry_adjustments', value=existing_count)
+
+            # adjust order price
+            return current_candle['sma_200']
+
+        # default: maintain existing order
+        return current_order_rate
+
+    def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
+
+        entry_adjustment_count = trade.get_kval(key='num_entry_adjustments')
+        trade_entry_type = trade.get_kval(key='entry_type')
+        if entry_adjustment_count is None:
+            if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
+                return True, 'exit_1'
+        else
+            if entry_adjustment_count > 0 and if current_profit > 0.05:
+                return True, 'exit_2'
+            if trade_entry_type == 'breakout' and current_profit > 0.1:
+                return True, 'exit_3
+
+        return False, None
+```
+
+!!! Note
+    It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
+
+!!! Warning
+    If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
+
 ## Dataframe access
 
 You may access dataframe in various strategy functions by querying it from dataprovider.

From abda02572b4248391dc2a9f4b9bc98f095735beb Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 5 Jun 2022 12:18:07 +0300
Subject: [PATCH 04/76] Fix KeyValue __repr__.

---
 freqtrade/persistence/keyvalue.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py
index 60fa903a1..2ed64f255 100644
--- a/freqtrade/persistence/keyvalue.py
+++ b/freqtrade/persistence/keyvalue.py
@@ -38,8 +38,8 @@ class KeyValue(_DECL_BASE):
                        if self.created_at is not None else None)
         update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
                        if self.updated_at is not None else None)
-        return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ',
-                f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ',
+        return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ' +
+                f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
                 f'updated={update_time})')
 
     @staticmethod

From be169a23f4e3ed307871a45635b3da89f347a133 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Mon, 13 Jun 2022 20:00:21 +0300
Subject: [PATCH 05/76] Add a new session for KeyValues.

---
 freqtrade/persistence/models.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index f2d75fec7..5ba0a28bd 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -58,7 +58,8 @@ def init_db(db_url: str) -> None:
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
-    KeyValue.query = Trade._session.query_property()
+    KeyValue._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
+    KeyValue.query = KeyValue._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)

From f3dee5ec4f98592c1363f5c53c6f375f1476f4fd Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Mon, 13 Jun 2022 20:02:06 +0300
Subject: [PATCH 06/76] Update handling for query_kv when no Key is supplied.

---
 freqtrade/persistence/keyvalue.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py
index 2ed64f255..d3d1454b7 100644
--- a/freqtrade/persistence/keyvalue.py
+++ b/freqtrade/persistence/keyvalue.py
@@ -49,9 +49,9 @@ class KeyValue(_DECL_BASE):
         return will be for generic values not tied to a trade
         :param trade_id: id of the Trade
         """
-        key = key if key is not None else "%"
-
-        filters = [KeyValue.ft_trade_id == trade_id if trade_id is not None else 0,
-                   KeyValue.kv_key.ilike(key)]
+        filters = []
+        filters.append(KeyValue.ft_trade_id == trade_id if trade_id is not None else 0)
+        if key is not None:
+            filters.append(KeyValue.kv_key.ilike(key))
 
         return KeyValue.query.filter(*filters)

From c719860a164a95a0718bc4437476396198e5b093 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Mon, 13 Jun 2022 20:03:22 +0300
Subject: [PATCH 07/76] get_kval() -> get_kvals(). Update docs also.

---
 docs/strategy-advanced.md            | 10 +++++-----
 freqtrade/persistence/trade_model.py | 13 +++++++------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 45961c59d..765dd3fab 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -43,7 +43,7 @@ class AwesomeStrategy(IStrategy):
 ## Storing information (Persistent)
 
 Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade.
-Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kval(key='my_key')`.
+Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kvals(key='my_key')`.
 Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle.
 For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string.
 
@@ -57,7 +57,7 @@ class AwesomeStrategy(IStrategy):
         for trade in Trade.get_open_order_trades():
             fills = trade.select_filled_orders(trade.entry_side)
             if trade.pair == 'ETH/USDT':
-                trade_entry_type = trade.get_kval(key='entry_type')
+                trade_entry_type = trade.get_kvals(key='entry_type').kv_value
                 if trade_entry_type is None:
                     trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
                 elif fills > 1:
@@ -73,7 +73,7 @@ class AwesomeStrategy(IStrategy):
             dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
             current_candle = dataframe.iloc[-1].squeeze()
             # store information about entry adjustment
-            existing_count = trade.get_kval(key='num_entry_adjustments')
+            existing_count = trade.get_kvals(key='num_entry_adjustments').kv_value
             if not existing_count:
                 existing_count = 1
             else:
@@ -88,8 +88,8 @@ class AwesomeStrategy(IStrategy):
 
     def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
 
-        entry_adjustment_count = trade.get_kval(key='num_entry_adjustments')
-        trade_entry_type = trade.get_kval(key='entry_type')
+        entry_adjustment_count = trade.get_kvals(key='num_entry_adjustments').kv_value
+        trade_entry_type = trade.get_kvals(key='entry_type').kv_value
         if entry_adjustment_count is None:
             if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
                 return True, 'exit_1'
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 83d400412..ce9fde59e 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -923,7 +923,7 @@ class LocalTrade():
     def set_kval(self, key: str, value: Any) -> None:
         KeyValues.set_kval(key=key, value=value, trade_id=self.id)
 
-    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
+    def get_kvals(self, key: Optional[str]) -> List[KeyValue]:
         return KeyValues.get_kval(key=key, trade_id=self.id)
 
     @property
@@ -1127,12 +1127,13 @@ class Trade(_DECL_BASE, LocalTrade):
         for order in self.orders:
             Order.query.session.delete(order)
 
-        for kval in self.keyvalues:
-            KeyValue.query.session.delete(kval)
-
         Trade.query.session.delete(self)
         Trade.commit()
 
+        for kval in self.keyvalues:
+            KeyValue.query.session.delete(kval)
+        KeyValue.query.session.commit()
+
     @staticmethod
     def commit():
         Trade.query.session.commit()
@@ -1409,5 +1410,5 @@ class Trade(_DECL_BASE, LocalTrade):
     def set_kval(self, key: str, value: Any) -> None:
         super().set_kval(key=key, value=value)
 
-    def get_kval(self, key: Optional[str]) -> List[KeyValue]:
-        return super().get_kval(key=key)
+    def get_kvals(self, key: Optional[str]) -> List[KeyValue]:
+        return super().get_kvals(key=key)

From 4f799cc9db8ed061af8385a775f9c1df67e421f0 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Mon, 13 Jun 2022 20:04:14 +0300
Subject: [PATCH 08/76] Add /list_kvals command for TG and underlying RPC.

---
 freqtrade/rpc/rpc.py           | 20 +++++++++++++
 freqtrade/rpc/telegram.py      | 51 +++++++++++++++++++++++++++++++++-
 tests/rpc/test_rpc_telegram.py |  2 +-
 3 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index a98e3f96d..929ab4150 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -843,6 +843,26 @@ class RPC:
                 'cancel_order_count': c_count,
             }
 
+    def _rpc_list_kvals(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
+        # Query for trade
+        trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
+        if trade is None:
+            return []
+        # Query keyvals
+        keyvals = trade.get_kvals(key=key)
+        return [
+            {
+                'id': kval.id,
+                'ft_trade_id': kval.ft_trade_id,
+                'kv_key': kval.kv_key,
+                'kv_type': kval.kv_type,
+                'kv_value': kval.kv_value,
+                'created_at': kval.created_at,
+                'updated_at': kval.updated_at
+            }
+            for kval in keyvals
+        ]
+
     def _rpc_performance(self) -> List[Dict[str, Any]]:
         """
         Handler for performance.
diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index e456b1eef..f5bed167d 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -182,6 +182,7 @@ class Telegram(RPCHandler):
             CommandHandler('health', self._health),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
+            CommandHandler('list_kvals', self._list_kvals),
         ]
         callbacks = [
             CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -1459,7 +1460,9 @@ class Telegram(RPCHandler):
             "*/stats:* `Shows Wins / losses by Sell reason as well as "
             "Avg. holding durationsfor buys and sells.`\n"
             "*/help:* `This help message`\n"
-            "*/version:* `Show version`"
+            "*/version:* `Show version`\n"
+            "*/list_kvals <trade_id> <key>:* `List key-value for Trade ID and Key combo.`\n"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
             )
 
         self._send_msg(message, parse_mode=ParseMode.MARKDOWN)
@@ -1539,6 +1542,52 @@ class Telegram(RPCHandler):
             f"*Current state:* `{val['state']}`"
         )
 
+    @authorized_only
+    def _list_kvals(self, update: Update, context: CallbackContext) -> None:
+        """
+        Handler for /list_kvals <id> <key>.
+        List keyvalues for specified trade (and key if supplied).
+        :param bot: telegram bot
+        :param update: message update
+        :return: None
+        """
+        try:
+            if not context.args or len(context.args) == 0:
+                raise RPCException("Trade-id not set.")
+            trade_id = int(context.args[0])
+            key = None if len(context.args) < 2 else str(context.args[1])
+
+            results = self._rpc._rpc_list_kvals(trade_id, key)
+            logger.warning(len(results))
+            logger.warning(results)
+            messages = []
+            if len(results) > 0:
+                messages = ['Found key-value pair' + 's:  \n' if key is None else ':  \n']
+                for result in results:
+                    lines = [
+                        f"*Key:* `{result['kv_key']}`",
+                        f"*ID:* `{result['id']}`",
+                        f"*Trade ID:* `{result['ft_trade_id']}`",
+                        f"*Type:* `{result['kv_type']}`",
+                        f"*Value:* `{result['kv_value']}`",
+                        f"*Create Date:* `{result['created_at']}`",
+                        f"*Update Date:* `{result['updated_at']}`"
+                    ]
+                    # Filter empty lines using list-comprehension
+                    messages.append("\n".join([line for line in lines if line]))
+                for msg in messages:
+                    logger.warning(msg)
+                    self._send_msg(msg)
+            else:
+                message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`"
+                logger.warning(message)
+                message += f" and Key: `{key}`." if key is not None else ""
+                logger.warning(message)
+                self._send_msg(message)
+
+        except RPCException as e:
+            self._send_msg(str(e))
+
     def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
                     reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
         if reload_able:
diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py
index 2bc4fc5c3..ee0bac9e5 100644
--- a/tests/rpc/test_rpc_telegram.py
+++ b/tests/rpc/test_rpc_telegram.py
@@ -102,7 +102,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
                    "['count'], ['locks'], ['unlock', 'delete_locks'], "
                    "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], "
                    "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], "
-                   "['logs'], ['edge'], ['health'], ['help'], ['version']"
+                   "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_kvals']"
                    "]")
 
     assert log_has(message_str, caplog)

From 3ad8111d118437d697a95543b4b6bf3b8f7dcab4 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Tue, 14 Jun 2022 13:26:45 +0300
Subject: [PATCH 09/76] Remove stray debug messages.

---
 freqtrade/rpc/telegram.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index a64242511..c29ec6daa 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1536,8 +1536,6 @@ class Telegram(RPCHandler):
             key = None if len(context.args) < 2 else str(context.args[1])
 
             results = self._rpc._rpc_list_kvals(trade_id, key)
-            logger.warning(len(results))
-            logger.warning(results)
             messages = []
             if len(results) > 0:
                 messages = ['Found key-value pair' + 's:  \n' if key is None else ':  \n']
@@ -1554,13 +1552,11 @@ class Telegram(RPCHandler):
                     # Filter empty lines using list-comprehension
                     messages.append("\n".join([line for line in lines if line]))
                 for msg in messages:
-                    logger.warning(msg)
                     self._send_msg(msg)
             else:
                 message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`"
                 logger.warning(message)
                 message += f" and Key: `{key}`." if key is not None else ""
-                logger.warning(message)
                 self._send_msg(message)
 
         except RPCException as e:

From 24b6ce450b3ce4c95bb808686bdbbba1ade3662a Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Tue, 14 Jun 2022 13:27:50 +0300
Subject: [PATCH 10/76] Further cleanup.

---
 freqtrade/rpc/telegram.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index c29ec6daa..665621975 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1555,7 +1555,6 @@ class Telegram(RPCHandler):
                     self._send_msg(msg)
             else:
                 message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`"
-                logger.warning(message)
                 message += f" and Key: `{key}`." if key is not None else ""
                 self._send_msg(message)
 

From 9fdb8b07accd04b4f629d075d52d6339e02ebaa2 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 15:56:50 +0300
Subject: [PATCH 11/76] Rename persistant storage infrastructure.

---
 docs/strategy-advanced.md                    |  14 +--
 freqtrade/optimize/backtesting.py            |   8 +-
 freqtrade/persistence/__init__.py            |   2 +-
 freqtrade/persistence/keyvalue.py            |  30 +++---
 freqtrade/persistence/keyvalue_middleware.py | 108 ++++++++++---------
 freqtrade/persistence/models.py              |   6 +-
 freqtrade/persistence/trade_model.py         |  30 +++---
 freqtrade/rpc/rpc.py                         |  22 ++--
 freqtrade/rpc/telegram.py                    |  18 ++--
 tests/rpc/test_rpc_telegram.py               |   2 +-
 10 files changed, 123 insertions(+), 117 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 765dd3fab..9cd05d4f6 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -43,7 +43,7 @@ class AwesomeStrategy(IStrategy):
 ## Storing information (Persistent)
 
 Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade.
-Using a trade object handle information can be stored using `trade_obj.set_kval(key='my_key', value=my_value)` and retrieved using `trade_obj.get_kvals(key='my_key')`.
+Using a trade object handle information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`.
 Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle.
 For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string.
 
@@ -57,12 +57,12 @@ class AwesomeStrategy(IStrategy):
         for trade in Trade.get_open_order_trades():
             fills = trade.select_filled_orders(trade.entry_side)
             if trade.pair == 'ETH/USDT':
-                trade_entry_type = trade.get_kvals(key='entry_type').kv_value
+                trade_entry_type = trade.get_custom_data(key='entry_type').kv_value
                 if trade_entry_type is None:
                     trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
                 elif fills > 1:
                     trade_entry_type = 'buy_up'
-                trade.set_kval(key='entry_type', value=trade_entry_type)
+                trade.set_custom_data(key='entry_type', value=trade_entry_type)
         return super().bot_loop_start(**kwargs)
 
     def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str,
@@ -73,12 +73,12 @@ class AwesomeStrategy(IStrategy):
             dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
             current_candle = dataframe.iloc[-1].squeeze()
             # store information about entry adjustment
-            existing_count = trade.get_kvals(key='num_entry_adjustments').kv_value
+            existing_count = trade.get_custom_data(key='num_entry_adjustments').kv_value
             if not existing_count:
                 existing_count = 1
             else:
                 existing_count += 1
-            trade.set_kval(key='num_entry_adjustments', value=existing_count)
+            trade.set_custom_data(key='num_entry_adjustments', value=existing_count)
 
             # adjust order price
             return current_candle['sma_200']
@@ -88,8 +88,8 @@ class AwesomeStrategy(IStrategy):
 
     def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
 
-        entry_adjustment_count = trade.get_kvals(key='num_entry_adjustments').kv_value
-        trade_entry_type = trade.get_kvals(key='entry_type').kv_value
+        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments').kv_value
+        trade_entry_type = trade.get_custom_data(key='entry_type').kv_value
         if entry_adjustment_count is None:
             if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
                 return True, 'exit_1'
diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 14cc8d2ef..3071fb019 100755
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -30,7 +30,7 @@ from freqtrade.optimize.bt_progress import BTProgress
 from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
                                                  store_backtest_signal_candles,
                                                  store_backtest_stats)
-from freqtrade.persistence import KeyValues, LocalTrade, Order, PairLocks, Trade
+from freqtrade.persistence import CustomDataWrapper, LocalTrade, Order, PairLocks, Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -151,7 +151,7 @@ class Backtesting:
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True
-        KeyValues.use_db = True  # ???
+        CustomDataWrapper.use_db = True
 
     def init_backtest_detail(self):
         # Load detail timeframe if specified
@@ -300,8 +300,8 @@ class Backtesting:
         Trade.use_db = False
         PairLocks.reset_locks()
         Trade.reset_trades()
-        KeyValues.use_db = False
-        KeyValues.reset_keyvalues()
+        CustomDataWrapper.use_db = False
+        CustomDataWrapper.reset_custom_data()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0
diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py
index 635445e40..12cb68a10 100644
--- a/freqtrade/persistence/__init__.py
+++ b/freqtrade/persistence/__init__.py
@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 
-from freqtrade.persistence.keyvalue_middleware import KeyValues
+from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper
 from freqtrade.persistence.models import cleanup_db, init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade
diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/keyvalue.py
index d3d1454b7..1f85467dd 100644
--- a/freqtrade/persistence/keyvalue.py
+++ b/freqtrade/persistence/keyvalue.py
@@ -8,28 +8,28 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.persistence.base import _DECL_BASE
 
 
-class KeyValue(_DECL_BASE):
+class CustomData(_DECL_BASE):
     """
-    KeyValue database model
+    CustomData database model
     Keeps records of metadata as key/value store
     for trades or global persistant values
     One to many relationship with Trades:
       - One trade can have many metadata entries
       - One metadata entry can only be associated with one Trade
     """
-    __tablename__ = 'keyvalue'
+    __tablename__ = 'trade_custom_data'
     # Uniqueness should be ensured over pair, order_id
     # its likely that order_id is unique per Pair on some exchanges.
-    __table_args__ = (UniqueConstraint('ft_trade_id', 'kv_key', name="_trade_id_kv_key"),)
+    __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
 
     id = Column(Integer, primary_key=True)
     ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0)
 
-    trade = relationship("Trade", back_populates="keyvalues")
+    trade = relationship("Trade", back_populates="custom_data")
 
-    kv_key = Column(String(255), nullable=False)
-    kv_type = Column(String(25), nullable=False)
-    kv_value = Column(Text, nullable=False)
+    cd_key = Column(String(255), nullable=False)
+    cd_type = Column(String(25), nullable=False)
+    cd_value = Column(Text, nullable=False)
     created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
     updated_at = Column(DateTime, nullable=True)
 
@@ -38,20 +38,20 @@ class KeyValue(_DECL_BASE):
                        if self.created_at is not None else None)
         update_time = (self.updated_at.strftime(DATETIME_PRINT_FORMAT)
                        if self.updated_at is not None else None)
-        return (f'KeyValue(id={self.id}, key={self.kv_key}, type={self.kv_type}, ' +
-                f'value={self.kv_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
+        return (f'CustomData(id={self.id}, key={self.cd_key}, type={self.cd_type}, ' +
+                f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
                 f'updated={update_time})')
 
     @staticmethod
-    def query_kv(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
+    def query_cd(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
         """
-        Get all keyvalues, if  trade_id is not specified
+        Get all CustomData, if trade_id is not specified
         return will be for generic values not tied to a trade
         :param trade_id: id of the Trade
         """
         filters = []
-        filters.append(KeyValue.ft_trade_id == trade_id if trade_id is not None else 0)
+        filters.append(CustomData.ft_trade_id == trade_id if trade_id is not None else 0)
         if key is not None:
-            filters.append(KeyValue.kv_key.ilike(key))
+            filters.append(CustomData.cd_key.ilike(key))
 
-        return KeyValue.query.filter(*filters)
+        return CustomData.query.filter(*filters)
diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/keyvalue_middleware.py
index 8248143ce..0f3c745ad 100644
--- a/freqtrade/persistence/keyvalue_middleware.py
+++ b/freqtrade/persistence/keyvalue_middleware.py
@@ -3,57 +3,63 @@ import logging
 from datetime import datetime
 from typing import Any, List, Optional
 
-from freqtrade.persistence.keyvalue import KeyValue
+from freqtrade.persistence.keyvalue import CustomData
 
 
 logger = logging.getLogger(__name__)
 
 
-class KeyValues():
+class CustomDataWrapper():
     """
-    KeyValues middleware class
+    CustomData middleware class
     Abstracts the database layer away so it becomes optional - which will be necessary to support
     backtesting and hyperopt in the future.
     """
 
     use_db = True
-    kvals: List[KeyValue] = []
+    custom_data: List[CustomData] = []
     unserialized_types = ['bool', 'float', 'int', 'str']
 
     @staticmethod
-    def reset_keyvalues() -> None:
+    def reset_custom_data() -> None:
         """
         Resets all key-value pairs. Only active for backtesting mode.
         """
-        if not KeyValues.use_db:
-            KeyValues.kvals = []
+        if not CustomDataWrapper.use_db:
+            CustomDataWrapper.custom_data = []
 
     @staticmethod
-    def get_kval(key: Optional[str] = None, trade_id: Optional[int] = None) -> List[KeyValue]:
+    def get_custom_data(key: Optional[str] = None,
+                        trade_id: Optional[int] = None) -> List[CustomData]:
         if trade_id is None:
             trade_id = 0
 
-        if KeyValues.use_db:
-            filtered_kvals = KeyValue.query_kv(trade_id=trade_id, key=key).all()
-            for index, kval in enumerate(filtered_kvals):
-                if kval.kv_type not in KeyValues.unserialized_types:
-                    kval.kv_value = json.loads(kval.kv_value)
-                filtered_kvals[index] = kval
-            return filtered_kvals
+        if CustomDataWrapper.use_db:
+            filtered_custom_data = CustomData.query_cd(trade_id=trade_id, key=key).all()
+            for index, data_entry in enumerate(filtered_custom_data):
+                if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
+                    data_entry.cd_value = json.loads(data_entry.cd_value)
+                filtered_custom_data[index] = data_entry
+            return filtered_custom_data
         else:
-            filtered_kvals = [kval for kval in KeyValues.kvals if (kval.ft_trade_id == trade_id)]
+            filtered_custom_data = [
+                data_entry for data_entry in CustomDataWrapper.custom_data
+                if (data_entry.ft_trade_id == trade_id)
+            ]
             if key is not None:
-                filtered_kvals = [
-                    kval for kval in filtered_kvals if (kval.kv_key.casefold() == key.casefold())]
-            return filtered_kvals
+                filtered_custom_data = [
+                    data_entry for data_entry in filtered_custom_data
+                    if (data_entry.cd_key.casefold() == key.casefold())
+                ]
+            return filtered_custom_data
 
     @staticmethod
-    def set_kval(key: str, value: Any, trade_id: Optional[int] = None) -> None:
+    def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
 
         value_type = type(value).__name__
         value_db = None
 
-        if value_type not in KeyValues.unserialized_types:
+        if value_type not in CustomDataWrapper.unserialized_types:
             try:
                 value_db = json.dumps(value)
             except TypeError as e:
@@ -64,44 +70,44 @@ class KeyValues():
         if trade_id is None:
             trade_id = 0
 
-        kvals = KeyValues.get_kval(key=key, trade_id=trade_id)
-        if kvals:
-            kv = kvals[0]
-            kv.kv_value = value
-            kv.updated_at = datetime.utcnow()
+        custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
+        if custom_data:
+            data_entry = custom_data[0]
+            data_entry.cd_value = value
+            data_entry.updated_at = datetime.utcnow()
         else:
-            kv = KeyValue(
-                ft_trade_id=trade_id,
-                kv_key=key,
-                kv_type=value_type,
-                kv_value=value,
-                created_at=datetime.utcnow()
+            data_entry = CustomData(
+                            ft_trade_id=trade_id,
+                            cd_key=key,
+                            cd_type=value_type,
+                            cd_value=value,
+                            created_at=datetime.utcnow()
             )
 
-        if KeyValues.use_db and value_db is not None:
-            kv.kv_value = value_db
-            KeyValue.query.session.add(kv)
-            KeyValue.query.session.commit()
-        elif not KeyValues.use_db:
-            kv_index = -1
-            for index, kval in enumerate(KeyValues.kvals):
-                if kval.ft_trade_id == trade_id and kval.kv_key == key:
-                    kv_index = index
+        if CustomDataWrapper.use_db and value_db is not None:
+            data_entry.cd_value = value_db
+            CustomData.query.session.add(data_entry)
+            CustomData.query.session.commit()
+        elif not CustomDataWrapper.use_db:
+            cd_index = -1
+            for index, data_entry in enumerate(CustomDataWrapper.custom_data):
+                if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
+                    cd_index = index
                     break
 
-            if kv_index >= 0:
-                kval.kv_type = value_type
-                kval.value = value
-                kval.updated_at = datetime.utcnow()
+            if cd_index >= 0:
+                data_entry.cd_type = value_type
+                data_entry.value = value
+                data_entry.updated_at = datetime.utcnow()
 
-                KeyValues.kvals[kv_index] = kval
+                CustomDataWrapper.custom_data[cd_index] = data_entry
             else:
-                KeyValues.kvals.append(kv)
+                CustomDataWrapper.custom_data.append(data_entry)
 
     @staticmethod
-    def get_all_kvals() -> List[KeyValue]:
+    def get_all_custom_data() -> List[CustomData]:
 
-        if KeyValues.use_db:
-            return KeyValue.query.all()
+        if CustomDataWrapper.use_db:
+            return CustomData.query.all()
         else:
-            return KeyValues.kvals
+            return CustomDataWrapper.custom_data
diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index 5ba0a28bd..a4c01b119 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import _DECL_BASE
-from freqtrade.persistence.keyvalue import KeyValue
+from freqtrade.persistence.keyvalue import CustomData
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -58,8 +58,8 @@ def init_db(db_url: str) -> None:
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
-    KeyValue._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
-    KeyValue.query = KeyValue._session.query_property()
+    CustomData._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
+    CustomData.query = CustomData._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 53abe638b..ac7ba4833 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -15,8 +15,8 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
-from freqtrade.persistence.keyvalue import KeyValue
-from freqtrade.persistence.keyvalue_middleware import KeyValues
+from freqtrade.persistence.keyvalue import CustomData
+from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper
 
 
 logger = logging.getLogger(__name__)
@@ -240,7 +240,7 @@ class LocalTrade():
     id: int = 0
 
     orders: List[Order] = []
-    keyvalues: List[KeyValue] = []
+    custom_data: List[CustomData] = []
 
     exchange: str = ''
     pair: str = ''
@@ -880,11 +880,11 @@ class LocalTrade():
                 or (o.ft_is_open is True and o.status is not None)
                 ]
 
-    def set_kval(self, key: str, value: Any) -> None:
-        KeyValues.set_kval(key=key, value=value, trade_id=self.id)
+    def set_custom_data(self, key: str, value: Any) -> None:
+        CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
 
-    def get_kvals(self, key: Optional[str]) -> List[KeyValue]:
-        return KeyValues.get_kval(key=key, trade_id=self.id)
+    def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
+        return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
 
     @property
     def nr_of_successful_entries(self) -> int:
@@ -1016,7 +1016,7 @@ class Trade(_DECL_BASE, LocalTrade):
     id = Column(Integer, primary_key=True)
 
     orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="joined")
-    keyvalues = relationship("KeyValue", order_by="KeyValue.id", cascade="all, delete-orphan")
+    custom_data = relationship("CustomData", order_by="CustomData.id", cascade="all, delete-orphan")
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
@@ -1090,9 +1090,9 @@ class Trade(_DECL_BASE, LocalTrade):
         Trade.query.session.delete(self)
         Trade.commit()
 
-        for kval in self.keyvalues:
-            KeyValue.query.session.delete(kval)
-        KeyValue.query.session.commit()
+        for entry in self.custom_data:
+            CustomData.query.session.delete(entry)
+        CustomData.query.session.commit()
 
     @staticmethod
     def commit():
@@ -1367,11 +1367,11 @@ class Trade(_DECL_BASE, LocalTrade):
             .order_by(desc('profit_sum')).first()
         return best_pair
 
-    def set_kval(self, key: str, value: Any) -> None:
-        super().set_kval(key=key, value=value)
+    def set_custom_data(self, key: str, value: Any) -> None:
+        super().set_custom_data(key=key, value=value)
 
-    def get_kvals(self, key: Optional[str]) -> List[KeyValue]:
-        return super().get_kvals(key=key)
+    def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
+        return super().get_custom_data(key=key)
 
     @staticmethod
     def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float:
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 608f51bcd..cee2007ff 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -805,24 +805,24 @@ class RPC:
                 'cancel_order_count': c_count,
             }
 
-    def _rpc_list_kvals(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
+    def _rpc_list_custom_data(self, trade_id: int, key: Optional[str]) -> List[Dict[str, Any]]:
         # Query for trade
         trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
         if trade is None:
             return []
-        # Query keyvals
-        keyvals = trade.get_kvals(key=key)
+        # Query custom_data
+        custom_data = trade.get_custom_data(key=key)
         return [
             {
-                'id': kval.id,
-                'ft_trade_id': kval.ft_trade_id,
-                'kv_key': kval.kv_key,
-                'kv_type': kval.kv_type,
-                'kv_value': kval.kv_value,
-                'created_at': kval.created_at,
-                'updated_at': kval.updated_at
+                'id': data_entry.id,
+                'ft_trade_id': data_entry.ft_trade_id,
+                'cd_key': data_entry.cd_key,
+                'cd_type': data_entry.cd_type,
+                'cd_value': data_entry.cd_value,
+                'created_at': data_entry.created_at,
+                'updated_at': data_entry.updated_at
             }
-            for kval in keyvals
+            for data_entry in custom_data
         ]
 
     def _rpc_performance(self) -> List[Dict[str, Any]]:
diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index ae4da9904..4af7c5d5d 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -192,7 +192,7 @@ class Telegram(RPCHandler):
             CommandHandler('health', self._health),
             CommandHandler('help', self._help),
             CommandHandler('version', self._version),
-            CommandHandler('list_kvals', self._list_kvals),
+            CommandHandler('list_custom_data', self._list_custom_data),
         ]
         callbacks = [
             CallbackQueryHandler(self._status_table, pattern='update_status_table'),
@@ -1453,7 +1453,7 @@ class Telegram(RPCHandler):
             "Avg. holding durationsfor buys and sells.`\n"
             "*/help:* `This help message`\n"
             "*/version:* `Show version`\n"
-            "*/list_kvals <trade_id> <key>:* `List key-value for Trade ID and Key combo.`\n"
+            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
             "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
             )
 
@@ -1535,10 +1535,10 @@ class Telegram(RPCHandler):
         )
 
     @authorized_only
-    def _list_kvals(self, update: Update, context: CallbackContext) -> None:
+    def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
         """
-        Handler for /list_kvals <id> <key>.
-        List keyvalues for specified trade (and key if supplied).
+        Handler for /list_custom_data <id> <key>.
+        List custom_data for specified trade (and key if supplied).
         :param bot: telegram bot
         :param update: message update
         :return: None
@@ -1549,17 +1549,17 @@ class Telegram(RPCHandler):
             trade_id = int(context.args[0])
             key = None if len(context.args) < 2 else str(context.args[1])
 
-            results = self._rpc._rpc_list_kvals(trade_id, key)
+            results = self._rpc._rpc_list_custom_data(trade_id, key)
             messages = []
             if len(results) > 0:
                 messages = ['Found key-value pair' + 's:  \n' if key is None else ':  \n']
                 for result in results:
                     lines = [
-                        f"*Key:* `{result['kv_key']}`",
+                        f"*Key:* `{result['cd_key']}`",
                         f"*ID:* `{result['id']}`",
                         f"*Trade ID:* `{result['ft_trade_id']}`",
-                        f"*Type:* `{result['kv_type']}`",
-                        f"*Value:* `{result['kv_value']}`",
+                        f"*Type:* `{result['cd_type']}`",
+                        f"*Value:* `{result['cd_value']}`",
                         f"*Create Date:* `{result['created_at']}`",
                         f"*Update Date:* `{result['updated_at']}`"
                     ]
diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py
index 789d10a02..39e33a8e3 100644
--- a/tests/rpc/test_rpc_telegram.py
+++ b/tests/rpc/test_rpc_telegram.py
@@ -103,7 +103,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
                    "['count'], ['locks'], ['unlock', 'delete_locks'], "
                    "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], "
                    "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], "
-                   "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_kvals']"
+                   "['logs'], ['edge'], ['health'], ['help'], ['version'], ['list_custom_data']"
                    "]")
 
     assert log_has(message_str, caplog)

From 365527508bbded7a6fb42e0d3351c018c682820a Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 15:59:21 +0300
Subject: [PATCH 12/76] Rename files.

---
 freqtrade/persistence/{keyvalue.py => custom_data.py}             | 0
 .../{keyvalue_middleware.py => custom_data_middleware.py}         | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename freqtrade/persistence/{keyvalue.py => custom_data.py} (100%)
 rename freqtrade/persistence/{keyvalue_middleware.py => custom_data_middleware.py} (100%)

diff --git a/freqtrade/persistence/keyvalue.py b/freqtrade/persistence/custom_data.py
similarity index 100%
rename from freqtrade/persistence/keyvalue.py
rename to freqtrade/persistence/custom_data.py
diff --git a/freqtrade/persistence/keyvalue_middleware.py b/freqtrade/persistence/custom_data_middleware.py
similarity index 100%
rename from freqtrade/persistence/keyvalue_middleware.py
rename to freqtrade/persistence/custom_data_middleware.py

From ce9d9d7e60b03566ecfab0f5eae6bf72eff89c01 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 16:02:24 +0300
Subject: [PATCH 13/76] Finish renaming persistant storage infrastructure.

---
 freqtrade/persistence/__init__.py               | 2 +-
 freqtrade/persistence/custom_data_middleware.py | 2 +-
 freqtrade/persistence/models.py                 | 2 +-
 freqtrade/persistence/trade_model.py            | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py
index 12cb68a10..bf0a8dcbf 100644
--- a/freqtrade/persistence/__init__.py
+++ b/freqtrade/persistence/__init__.py
@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 
-from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper
+from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
 from freqtrade.persistence.models import cleanup_db, init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
index 0f3c745ad..2fe4bd931 100644
--- a/freqtrade/persistence/custom_data_middleware.py
+++ b/freqtrade/persistence/custom_data_middleware.py
@@ -3,7 +3,7 @@ import logging
 from datetime import datetime
 from typing import Any, List, Optional
 
-from freqtrade.persistence.keyvalue import CustomData
+from freqtrade.persistence.custom_data import CustomData
 
 
 logger = logging.getLogger(__name__)
diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index a4c01b119..16076adb9 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import _DECL_BASE
-from freqtrade.persistence.keyvalue import CustomData
+from freqtrade.persistence.custom_data import CustomData
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index ac7ba4833..582e91d3d 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -15,8 +15,8 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
-from freqtrade.persistence.keyvalue import CustomData
-from freqtrade.persistence.keyvalue_middleware import CustomDataWrapper
+from freqtrade.persistence.custom_data import CustomData
+from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
 
 
 logger = logging.getLogger(__name__)

From c8ba8106e668ea92c6f57d577f2d9e4ae750b4b3 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 17:24:13 +0300
Subject: [PATCH 14/76] Update telegram reporting.

---
 freqtrade/rpc/telegram.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 4af7c5d5d..6bd68fd3d 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1552,7 +1552,9 @@ class Telegram(RPCHandler):
             results = self._rpc._rpc_list_custom_data(trade_id, key)
             messages = []
             if len(results) > 0:
-                messages = ['Found key-value pair' + 's:  \n' if key is None else ':  \n']
+                messages.append(
+                    'Found custom-data entr' + ('ies: ' if len(results) > 1 else 'y: ')
+                )
                 for result in results:
                     lines = [
                         f"*Key:* `{result['cd_key']}`",
@@ -1568,7 +1570,7 @@ class Telegram(RPCHandler):
                 for msg in messages:
                     self._send_msg(msg)
             else:
-                message = f"Didn't find any key-value pairs for Trade ID: `{trade_id}`"
+                message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"
                 message += f" and Key: `{key}`." if key is not None else ""
                 self._send_msg(message)
 

From 8494bea64f0b7534553b28c15bf1b5a1a791e383 Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 19:59:14 +0300
Subject: [PATCH 15/76] Handle max message length.

---
 freqtrade/rpc/telegram.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 6bd68fd3d..6b3ccaac8 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1568,6 +1568,10 @@ class Telegram(RPCHandler):
                     # Filter empty lines using list-comprehension
                     messages.append("\n".join([line for line in lines if line]))
                 for msg in messages:
+                    if len(msg) > MAX_TELEGRAM_MESSAGE_LENGTH:
+                        msg = "Message dropped because length exceeds "
+                        msg += f"maximum allowed characters: {MAX_TELEGRAM_MESSAGE_LENGTH}"
+                        logger.warning(msg)
                     self._send_msg(msg)
             else:
                 message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"

From c420304b33b94f67533864345201893ea28f2b9b Mon Sep 17 00:00:00 2001
From: eSeR1805 <costianuv@gmail.com>
Date: Sun, 19 Jun 2022 20:03:56 +0300
Subject: [PATCH 16/76] Delete custom data before the trade.

---
 freqtrade/persistence/trade_model.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 582e91d3d..7e0314738 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1087,12 +1087,12 @@ class Trade(_DECL_BASE, LocalTrade):
         for order in self.orders:
             Order.query.session.delete(order)
 
-        Trade.query.session.delete(self)
-        Trade.commit()
-
         for entry in self.custom_data:
             CustomData.query.session.delete(entry)
+
         CustomData.query.session.commit()
+        Trade.query.session.delete(self)
+        Trade.commit()
 
     @staticmethod
     def commit():

From 8f9f4b40cdcad855cec449f9364a449a22cfad67 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 7 Feb 2024 07:21:16 +0100
Subject: [PATCH 17/76] Update model to new sqlalchemy version

---
 freqtrade/persistence/custom_data.py          | 29 ++++++++++---------
 .../persistence/custom_data_middleware.py     | 10 ++++---
 freqtrade/persistence/trade_model.py          |  3 +-
 freqtrade/rpc/telegram.py                     |  8 ++---
 4 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index 1f85467dd..beae8c478 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -1,14 +1,15 @@
 from datetime import datetime
-from typing import Optional
+from typing import ClassVar, Optional
 
-from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
-from sqlalchemy.orm import Query, relationship
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
+from sqlalchemy.orm import Mapped, Query, mapped_column, relationship
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase, SessionType
+from freqtrade.util import dt_now
 
 
-class CustomData(_DECL_BASE):
+class CustomData(ModelBase):
     """
     CustomData database model
     Keeps records of metadata as key/value store
@@ -18,20 +19,22 @@ class CustomData(_DECL_BASE):
       - One metadata entry can only be associated with one Trade
     """
     __tablename__ = 'trade_custom_data'
+    session: ClassVar[SessionType]
+
     # Uniqueness should be ensured over pair, order_id
     # its likely that order_id is unique per Pair on some exchanges.
     __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
 
-    id = Column(Integer, primary_key=True)
-    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True, default=0)
+    id = mapped_column(Integer, primary_key=True)
+    ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True, default=0)
 
     trade = relationship("Trade", back_populates="custom_data")
 
-    cd_key = Column(String(255), nullable=False)
-    cd_type = Column(String(25), nullable=False)
-    cd_value = Column(Text, nullable=False)
-    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
-    updated_at = Column(DateTime, nullable=True)
+    cd_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    cd_type: Mapped[str] = mapped_column(String(25), nullable=False)
+    cd_value: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now)
+    updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     def __repr__(self):
         create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
@@ -54,4 +57,4 @@ class CustomData(_DECL_BASE):
         if key is not None:
             filters.append(CustomData.cd_key.ilike(key))
 
-        return CustomData.query.filter(*filters)
+        return CustomData.session.scalars(select(CustomData))
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
index 64e8a47fd..cf7b83abc 100644
--- a/freqtrade/persistence/custom_data_middleware.py
+++ b/freqtrade/persistence/custom_data_middleware.py
@@ -3,6 +3,8 @@ import logging
 from datetime import datetime
 from typing import Any, List, Optional
 
+from sqlalchemy import select
+
 from freqtrade.persistence.custom_data import CustomData
 
 
@@ -86,8 +88,8 @@ class CustomDataWrapper:
 
         if CustomDataWrapper.use_db and value_db is not None:
             data_entry.cd_value = value_db
-            CustomData.query.session.add(data_entry)
-            CustomData.query.session.commit()
+            CustomData.session.add(data_entry)
+            CustomData.session.commit()
         elif not CustomDataWrapper.use_db:
             cd_index = -1
             for index, data_entry in enumerate(CustomDataWrapper.custom_data):
@@ -97,7 +99,7 @@ class CustomDataWrapper:
 
             if cd_index >= 0:
                 data_entry.cd_type = value_type
-                data_entry.value = value
+                data_entry.cd_value = value
                 data_entry.updated_at = datetime.utcnow()
 
                 CustomDataWrapper.custom_data[cd_index] = data_entry
@@ -108,6 +110,6 @@ class CustomDataWrapper:
     def get_all_custom_data() -> List[CustomData]:
 
         if CustomDataWrapper.use_db:
-            return CustomData.query.all()
+            return CustomData.session.scalars(select(CustomData)).all()
         else:
             return CustomDataWrapper.custom_data
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index a3f71ee4f..d96378bf2 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1467,7 +1467,8 @@ class Trade(ModelBase, LocalTrade):
     orders: Mapped[List[Order]] = relationship(
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
-    custom_data = relationship("CustomData", order_by="CustomData.id", cascade="all, delete-orphan")
+    custom_data: Mapped[List[CustomData]] = relationship(
+        "CustomData", order_by="CustomData.id", cascade="all, delete-orphan")  # type: ignore
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
     pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 4bdb09f01..4d91d1bdb 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1770,7 +1770,7 @@ class Telegram(RPCHandler):
         )
 
     @authorized_only
-    def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
+    async def _list_custom_data(self, update: Update, context: CallbackContext) -> None:
         """
         Handler for /list_custom_data <id> <key>.
         List custom_data for specified trade (and key if supplied).
@@ -1807,14 +1807,14 @@ class Telegram(RPCHandler):
                         msg = "Message dropped because length exceeds "
                         msg += f"maximum allowed characters: {MAX_MESSAGE_LENGTH}"
                         logger.warning(msg)
-                    self._send_msg(msg)
+                    await self._send_msg(msg)
             else:
                 message = f"Didn't find any custom-data entries for Trade ID: `{trade_id}`"
                 message += f" and Key: `{key}`." if key is not None else ""
-                self._send_msg(message)
+                await self._send_msg(message)
 
         except RPCException as e:
-            self._send_msg(str(e))
+            await self._send_msg(str(e))
 
     async def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
                           reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:

From 2393a9fecf04eb284078ec1d09e6b0113138d5f4 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 7 Feb 2024 19:06:41 +0100
Subject: [PATCH 18/76] Fix some minor test failures

---
 freqtrade/persistence/models.py | 4 ++--
 tests/rpc/test_rpc_telegram.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index 3f1661ee2..189b80fa6 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -79,8 +79,8 @@ def init_db(db_url: str) -> None:
     Order.session = Trade.session
     PairLock.session = Trade.session
     _KeyValueStoreModel.session = Trade.session
-    CustomData._session = scoped_session(sessionmaker(bind=engine, autoflush=True),
-                                         scopefunc=get_request_or_thread_id)
+    CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True),
+                                        scopefunc=get_request_or_thread_id)
 
     previous_tables = inspect(engine).get_table_names()
     ModelBase.metadata.create_all(engine)
diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py
index 110cd7819..45268b23b 100644
--- a/tests/rpc/test_rpc_telegram.py
+++ b/tests/rpc/test_rpc_telegram.py
@@ -150,7 +150,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
                    "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], "
                    "['bl_delete', 'blacklist_delete'], "
                    "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], "
-                   "['order']], ['list_custom_data']")
+                   "['order'], ['list_custom_data']]")
 
     assert log_has(message_str, caplog)
 

From 626c9041039212f7672dd3b5e769323de54f636f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 7 Feb 2024 19:28:06 +0100
Subject: [PATCH 19/76] Fix some issues with types

---
 freqtrade/persistence/custom_data.py            | 14 ++++++++------
 freqtrade/persistence/custom_data_middleware.py |  8 ++++----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index beae8c478..42b267e95 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -1,8 +1,8 @@
 from datetime import datetime
-from typing import ClassVar, Optional
+from typing import ClassVar, Optional, Self, Sequence
 
 from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
-from sqlalchemy.orm import Mapped, Query, mapped_column, relationship
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.persistence.base import ModelBase, SessionType
@@ -45,16 +45,18 @@ class CustomData(ModelBase):
                 f'value={self.cd_value}, trade_id={self.ft_trade_id}, created={create_time}, ' +
                 f'updated={update_time})')
 
-    @staticmethod
-    def query_cd(key: Optional[str] = None, trade_id: Optional[int] = None) -> Query:
+    @classmethod
+    def query_cd(cls, key: Optional[str] = None,
+                 trade_id: Optional[int] = None) -> Sequence['CustomData']:
         """
         Get all CustomData, if trade_id is not specified
         return will be for generic values not tied to a trade
         :param trade_id: id of the Trade
         """
         filters = []
-        filters.append(CustomData.ft_trade_id == trade_id if trade_id is not None else 0)
+        if trade_id is not None:
+            filters.append(CustomData.ft_trade_id == trade_id)
         if key is not None:
             filters.append(CustomData.cd_key.ilike(key))
 
-        return CustomData.session.scalars(select(CustomData))
+        return CustomData.session.scalars(select(CustomData)).all()
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
index cf7b83abc..2f99d9c75 100644
--- a/freqtrade/persistence/custom_data_middleware.py
+++ b/freqtrade/persistence/custom_data_middleware.py
@@ -37,11 +37,11 @@ class CustomDataWrapper:
             trade_id = 0
 
         if CustomDataWrapper.use_db:
-            filtered_custom_data = CustomData.query_cd(trade_id=trade_id, key=key).all()
-            for index, data_entry in enumerate(filtered_custom_data):
+            filtered_custom_data = []
+            for data_entry in CustomData.query_cd(trade_id=trade_id, key=key):
                 if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
                     data_entry.cd_value = json.loads(data_entry.cd_value)
-                filtered_custom_data[index] = data_entry
+                filtered_custom_data.append(data_entry)
             return filtered_custom_data
         else:
             filtered_custom_data = [
@@ -110,6 +110,6 @@ class CustomDataWrapper:
     def get_all_custom_data() -> List[CustomData]:
 
         if CustomDataWrapper.use_db:
-            return CustomData.session.scalars(select(CustomData)).all()
+            return list(CustomData.query_cd())
         else:
             return CustomDataWrapper.custom_data

From d49da763824aec1ad5fefd38aee938ef34e2a6b8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 18:17:48 +0100
Subject: [PATCH 20/76] Slighlty improve docs

---
 docs/strategy-advanced.md | 63 ++++++++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 24 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index b2a0d8431..69d4ced34 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -13,38 +13,46 @@ The call sequence of the methods described here is covered under [bot execution
 
 ## Storing information (Non-Persistent)
 
-Storing information can be accomplished by creating a new dictionary within the strategy class.
+!!! Warning "Deprecated"
+    This method of storing information is deprecated, and we do advise against using non-persistent storage.  
+    Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead.
 
-The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+    It's content has therefore be collapsed.
 
-```python
-class AwesomeStrategy(IStrategy):
-    # Create custom dictionary
-    custom_info = {}
+??? Abstract "Storing information"
+    Storing information can be accomplished by creating a new dictionary within the strategy class.
 
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        # Check if the entry already exists
-        if not metadata["pair"] in self.custom_info:
-            # Create empty entry for this pair
-            self.custom_info[metadata["pair"]] = {}
+    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
 
-        if "crosstime" in self.custom_info[metadata["pair"]]:
-            self.custom_info[metadata["pair"]]["crosstime"] += 1
-        else:
-            self.custom_info[metadata["pair"]]["crosstime"] = 1
-```
+    ```python
+    class AwesomeStrategy(IStrategy):
+        # Create custom dictionary
+        custom_info = {}
 
-!!! Warning
-    The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Check if the entry already exists
+            if not metadata["pair"] in self.custom_info:
+                # Create empty entry for this pair
+                self.custom_info[metadata["pair"]] = {}
 
-!!! Note
-    If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+            if "crosstime" in self.custom_info[metadata["pair"]]:
+                self.custom_info[metadata["pair"]]["crosstime"] += 1
+            else:
+                self.custom_info[metadata["pair"]]["crosstime"] = 1
+    ```
+
+    !!! Warning
+        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+
+    !!! Note
+        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
 
 ## Storing information (Persistent)
 
 Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade.
-Using a trade object handle information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`.
-Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object handle.
+
+Using a trade object, information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
+
 For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string.
 
 ```python
@@ -69,7 +77,13 @@ class AwesomeStrategy(IStrategy):
                            current_time: datetime, proposed_rate: float, current_order_rate: float,
                            entry_tag: Optional[str], side: str, **kwargs) -> float:
         # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair.
-        if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc and order.filled == 0.0:
+        if (
+            pair == 'BTC/USDT' 
+            and entry_tag == 'long_sma200' 
+            and side == 'long' 
+            and (current_time - timedelta(minutes=10)) > trade.open_date_utc 
+            and order.filled == 0.0
+        ):
             dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
             current_candle = dataframe.iloc[-1].squeeze()
             # store information about entry adjustment
@@ -104,8 +118,9 @@ class AwesomeStrategy(IStrategy):
 
 !!! Note
     It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
+    Storing big junks of data may lead to unintended side-effects, like a database becoming big pretty fast (and as a consequence, also slow).
 
-!!! Warning
+!!! Warning "Non-serializable data"
     If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
 
 ## Dataframe access

From 85930946195444937accc653770655788e9fd32e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 18:22:49 +0100
Subject: [PATCH 21/76] Ensure custom data access goes through the accessor
 functions

---
 freqtrade/persistence/trade_model.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index d96378bf2..b0b4a58dc 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1468,7 +1468,8 @@ class Trade(ModelBase, LocalTrade):
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
     custom_data: Mapped[List[CustomData]] = relationship(
-        "CustomData", order_by="CustomData.id", cascade="all, delete-orphan")  # type: ignore
+        "CustomData", order_by="CustomData.id", cascade="all, delete-orphan",
+        lazy="raise")  # type: ignore
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
     pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore

From c67e451fe177c00f8148d0d66915e957ab15b8a2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 18:24:26 +0100
Subject: [PATCH 22/76] Remove unused imports

---
 freqtrade/persistence/custom_data.py            | 2 +-
 freqtrade/persistence/custom_data_middleware.py | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index 42b267e95..004aa51df 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import ClassVar, Optional, Self, Sequence
+from typing import ClassVar, Optional, Sequence
 
 from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
 from sqlalchemy.orm import Mapped, mapped_column, relationship
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
index 2f99d9c75..e40491b28 100644
--- a/freqtrade/persistence/custom_data_middleware.py
+++ b/freqtrade/persistence/custom_data_middleware.py
@@ -3,8 +3,6 @@ import logging
 from datetime import datetime
 from typing import Any, List, Optional
 
-from sqlalchemy import select
-
 from freqtrade.persistence.custom_data import CustomData
 
 

From 83b22dedd5a6151edf67dcc0c96bcbf8207caccb Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 18:25:02 +0100
Subject: [PATCH 23/76] Fix non-reset of use_db

---
 freqtrade/persistence/usedb_context.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py
index 795388af0..193f7021d 100644
--- a/freqtrade/persistence/usedb_context.py
+++ b/freqtrade/persistence/usedb_context.py
@@ -12,6 +12,7 @@ def disable_database_use(timeframe: str) -> None:
     PairLocks.use_db = False
     PairLocks.timeframe = timeframe
     Trade.use_db = False
+    CustomDataWrapper.use_db = False
 
 
 def enable_database_use() -> None:

From 9699011cd9e607c4efacc30428c0ebe6f5048b67 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 18:28:29 +0100
Subject: [PATCH 24/76] Remove pointless wrapper

---
 freqtrade/persistence/trade_model.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index b0b4a58dc..ea03e2c29 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1882,12 +1882,6 @@ class Trade(ModelBase, LocalTrade):
 
         return best_pair
 
-    def set_custom_data(self, key: str, value: Any) -> None:
-        super().set_custom_data(key=key, value=value)
-
-    def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
-        return super().get_custom_data(key=key)
-
     @staticmethod
     def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float:
         """

From 7fd70b82fa61d8f1f68a8394f5b616eaff4861f5 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:05:13 +0100
Subject: [PATCH 25/76] custom_data: Simplify and fix a few things

---
 freqtrade/persistence/custom_data.py          |  2 +-
 .../persistence/custom_data_middleware.py     | 22 +++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index 004aa51df..cfe150967 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -59,4 +59,4 @@ class CustomData(ModelBase):
         if key is not None:
             filters.append(CustomData.cd_key.ilike(key))
 
-        return CustomData.session.scalars(select(CustomData)).all()
+        return CustomData.session.scalars(select(CustomData).filter(*filters)).all()
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
index e40491b28..acc65606b 100644
--- a/freqtrade/persistence/custom_data_middleware.py
+++ b/freqtrade/persistence/custom_data_middleware.py
@@ -1,9 +1,9 @@
 import json
 import logging
-from datetime import datetime
 from typing import Any, List, Optional
 
 from freqtrade.persistence.custom_data import CustomData
+from freqtrade.util import dt_now
 
 
 logger = logging.getLogger(__name__)
@@ -30,7 +30,7 @@ class CustomDataWrapper:
 
     @staticmethod
     def get_custom_data(key: Optional[str] = None,
-                        trade_id: Optional[int] = None) -> List[CustomData]:
+                        trade_id: Optional[int] = None) -> CustomData:
         if trade_id is None:
             trade_id = 0
 
@@ -73,15 +73,15 @@ class CustomDataWrapper:
         custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
         if custom_data:
             data_entry = custom_data[0]
-            data_entry.cd_value = value
-            data_entry.updated_at = datetime.utcnow()
+            data_entry.cd_value = value_db
+            data_entry.updated_at = dt_now()
         else:
             data_entry = CustomData(
-                            ft_trade_id=trade_id,
-                            cd_key=key,
-                            cd_type=value_type,
-                            cd_value=value,
-                            created_at=datetime.utcnow()
+                ft_trade_id=trade_id,
+                cd_key=key,
+                cd_type=value_type,
+                cd_value=value_db,
+                created_at=dt_now()
             )
 
         if CustomDataWrapper.use_db and value_db is not None:
@@ -97,8 +97,8 @@ class CustomDataWrapper:
 
             if cd_index >= 0:
                 data_entry.cd_type = value_type
-                data_entry.cd_value = value
-                data_entry.updated_at = datetime.utcnow()
+                data_entry.cd_value = value_db
+                data_entry.updated_at = dt_now()
 
                 CustomDataWrapper.custom_data[cd_index] = data_entry
             else:

From b7904b8e805b57a7cd6b588c203ee981229b0b5e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:14:37 +0100
Subject: [PATCH 26/76] Combine custom_data classes to one file

---
 freqtrade/persistence/__init__.py             |   2 +-
 freqtrade/persistence/custom_data.py          | 111 ++++++++++++++++-
 .../persistence/custom_data_middleware.py     | 113 ------------------
 freqtrade/persistence/trade_model.py          |  13 +-
 freqtrade/persistence/usedb_context.py        |   2 +-
 5 files changed, 118 insertions(+), 123 deletions(-)
 delete mode 100644 freqtrade/persistence/custom_data_middleware.py

diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py
index 5926f2ad3..d5584c22c 100644
--- a/freqtrade/persistence/__init__.py
+++ b/freqtrade/persistence/__init__.py
@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 
-from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.key_value_store import KeyStoreKeys, KeyValueStore
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index cfe150967..bf6056278 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -1,5 +1,7 @@
+import json
+import logging
 from datetime import datetime
-from typing import ClassVar, Optional, Sequence
+from typing import Any, ClassVar, List, Optional, Sequence
 
 from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, select
 from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -9,6 +11,9 @@ from freqtrade.persistence.base import ModelBase, SessionType
 from freqtrade.util import dt_now
 
 
+logger = logging.getLogger(__name__)
+
+
 class CustomData(ModelBase):
     """
     CustomData database model
@@ -60,3 +65,107 @@ class CustomData(ModelBase):
             filters.append(CustomData.cd_key.ilike(key))
 
         return CustomData.session.scalars(select(CustomData).filter(*filters)).all()
+
+
+class CustomDataWrapper:
+    """
+    CustomData middleware class
+    Abstracts the database layer away so it becomes optional - which will be necessary to support
+    backtesting and hyperopt in the future.
+    """
+
+    use_db = True
+    custom_data: List[CustomData] = []
+    unserialized_types = ['bool', 'float', 'int', 'str']
+
+    @staticmethod
+    def reset_custom_data() -> None:
+        """
+        Resets all key-value pairs. Only active for backtesting mode.
+        """
+        if not CustomDataWrapper.use_db:
+            CustomDataWrapper.custom_data = []
+
+    @staticmethod
+    def get_custom_data(key: Optional[str] = None,
+                        trade_id: Optional[int] = None) -> CustomData:
+        if trade_id is None:
+            trade_id = 0
+
+        if CustomDataWrapper.use_db:
+            filtered_custom_data = []
+            for data_entry in CustomData.query_cd(trade_id=trade_id, key=key):
+                if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
+                    data_entry.cd_value = json.loads(data_entry.cd_value)
+                filtered_custom_data.append(data_entry)
+            return filtered_custom_data
+        else:
+            filtered_custom_data = [
+                data_entry for data_entry in CustomDataWrapper.custom_data
+                if (data_entry.ft_trade_id == trade_id)
+            ]
+            if key is not None:
+                filtered_custom_data = [
+                    data_entry for data_entry in filtered_custom_data
+                    if (data_entry.cd_key.casefold() == key.casefold())
+                ]
+            return filtered_custom_data
+
+    @staticmethod
+    def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
+
+        value_type = type(value).__name__
+        value_db = None
+
+        if value_type not in CustomDataWrapper.unserialized_types:
+            try:
+                value_db = json.dumps(value)
+            except TypeError as e:
+                logger.warning(f"could not serialize {key} value due to {e}")
+        else:
+            value_db = str(value)
+
+        if trade_id is None:
+            trade_id = 0
+
+        custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
+        if custom_data:
+            data_entry = custom_data[0]
+            data_entry.cd_value = value_db
+            data_entry.updated_at = dt_now()
+        else:
+            data_entry = CustomData(
+                ft_trade_id=trade_id,
+                cd_key=key,
+                cd_type=value_type,
+                cd_value=value_db,
+                created_at=dt_now()
+            )
+
+        if CustomDataWrapper.use_db and value_db is not None:
+            data_entry.cd_value = value_db
+            CustomData.session.add(data_entry)
+            CustomData.session.commit()
+        elif not CustomDataWrapper.use_db:
+            cd_index = -1
+            for index, data_entry in enumerate(CustomDataWrapper.custom_data):
+                if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
+                    cd_index = index
+                    break
+
+            if cd_index >= 0:
+                data_entry.cd_type = value_type
+                data_entry.cd_value = value_db
+                data_entry.updated_at = dt_now()
+
+                CustomDataWrapper.custom_data[cd_index] = data_entry
+            else:
+                CustomDataWrapper.custom_data.append(data_entry)
+
+    @staticmethod
+    def get_all_custom_data() -> List[CustomData]:
+
+        if CustomDataWrapper.use_db:
+            return list(CustomData.query_cd())
+        else:
+            return CustomDataWrapper.custom_data
diff --git a/freqtrade/persistence/custom_data_middleware.py b/freqtrade/persistence/custom_data_middleware.py
deleted file mode 100644
index acc65606b..000000000
--- a/freqtrade/persistence/custom_data_middleware.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import json
-import logging
-from typing import Any, List, Optional
-
-from freqtrade.persistence.custom_data import CustomData
-from freqtrade.util import dt_now
-
-
-logger = logging.getLogger(__name__)
-
-
-class CustomDataWrapper:
-    """
-    CustomData middleware class
-    Abstracts the database layer away so it becomes optional - which will be necessary to support
-    backtesting and hyperopt in the future.
-    """
-
-    use_db = True
-    custom_data: List[CustomData] = []
-    unserialized_types = ['bool', 'float', 'int', 'str']
-
-    @staticmethod
-    def reset_custom_data() -> None:
-        """
-        Resets all key-value pairs. Only active for backtesting mode.
-        """
-        if not CustomDataWrapper.use_db:
-            CustomDataWrapper.custom_data = []
-
-    @staticmethod
-    def get_custom_data(key: Optional[str] = None,
-                        trade_id: Optional[int] = None) -> CustomData:
-        if trade_id is None:
-            trade_id = 0
-
-        if CustomDataWrapper.use_db:
-            filtered_custom_data = []
-            for data_entry in CustomData.query_cd(trade_id=trade_id, key=key):
-                if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
-                    data_entry.cd_value = json.loads(data_entry.cd_value)
-                filtered_custom_data.append(data_entry)
-            return filtered_custom_data
-        else:
-            filtered_custom_data = [
-                data_entry for data_entry in CustomDataWrapper.custom_data
-                if (data_entry.ft_trade_id == trade_id)
-            ]
-            if key is not None:
-                filtered_custom_data = [
-                    data_entry for data_entry in filtered_custom_data
-                    if (data_entry.cd_key.casefold() == key.casefold())
-                ]
-            return filtered_custom_data
-
-    @staticmethod
-    def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
-
-        value_type = type(value).__name__
-        value_db = None
-
-        if value_type not in CustomDataWrapper.unserialized_types:
-            try:
-                value_db = json.dumps(value)
-            except TypeError as e:
-                logger.warning(f"could not serialize {key} value due to {e}")
-        else:
-            value_db = str(value)
-
-        if trade_id is None:
-            trade_id = 0
-
-        custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
-        if custom_data:
-            data_entry = custom_data[0]
-            data_entry.cd_value = value_db
-            data_entry.updated_at = dt_now()
-        else:
-            data_entry = CustomData(
-                ft_trade_id=trade_id,
-                cd_key=key,
-                cd_type=value_type,
-                cd_value=value_db,
-                created_at=dt_now()
-            )
-
-        if CustomDataWrapper.use_db and value_db is not None:
-            data_entry.cd_value = value_db
-            CustomData.session.add(data_entry)
-            CustomData.session.commit()
-        elif not CustomDataWrapper.use_db:
-            cd_index = -1
-            for index, data_entry in enumerate(CustomDataWrapper.custom_data):
-                if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
-                    cd_index = index
-                    break
-
-            if cd_index >= 0:
-                data_entry.cd_type = value_type
-                data_entry.cd_value = value_db
-                data_entry.updated_at = dt_now()
-
-                CustomDataWrapper.custom_data[cd_index] = data_entry
-            else:
-                CustomDataWrapper.custom_data.append(data_entry)
-
-    @staticmethod
-    def get_all_custom_data() -> List[CustomData]:
-
-        if CustomDataWrapper.use_db:
-            return list(CustomData.query_cd())
-        else:
-            return CustomDataWrapper.custom_data
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index ea03e2c29..7487c72b3 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -23,8 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi
 from freqtrade.leverage import interest
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.base import ModelBase, SessionType
-from freqtrade.persistence.custom_data import CustomData
-from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
+from freqtrade.persistence.custom_data import CustomData, CustomDataWrapper
 from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts
 
 
@@ -345,7 +344,7 @@ class LocalTrade:
     id: int = 0
 
     orders: List[Order] = []
-    custom_data: List[CustomData] = []
+    custom_data: List[_CustomData] = []
 
     exchange: str = ''
     pair: str = ''
@@ -1209,7 +1208,7 @@ class LocalTrade:
     def set_custom_data(self, key: str, value: Any) -> None:
         CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
 
-    def get_custom_data(self, key: Optional[str]) -> List[CustomData]:
+    def get_custom_data(self, key: Optional[str]) -> List[_CustomData]:
         return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
 
     @property
@@ -1467,7 +1466,7 @@ class Trade(ModelBase, LocalTrade):
     orders: Mapped[List[Order]] = relationship(
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
-    custom_data: Mapped[List[CustomData]] = relationship(
+    custom_data: Mapped[List[_CustomData]] = relationship(
         "CustomData", order_by="CustomData.id", cascade="all, delete-orphan",
         lazy="raise")  # type: ignore
 
@@ -1574,9 +1573,9 @@ class Trade(ModelBase, LocalTrade):
             Order.session.delete(order)
 
         for entry in self.custom_data:
-            CustomData.session.delete(entry)
+            _CustomData.session.delete(entry)
 
-        CustomData.session.commit()
+        _CustomData.session.commit()
         Trade.session.delete(self)
         Trade.commit()
 
diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py
index 193f7021d..732f0b0f8 100644
--- a/freqtrade/persistence/usedb_context.py
+++ b/freqtrade/persistence/usedb_context.py
@@ -1,5 +1,5 @@
 
-from freqtrade.persistence.custom_data_middleware import CustomDataWrapper
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import Trade
 

From 8dda28351e9e6728877a169553431e3f94ad3092 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:25:26 +0100
Subject: [PATCH 27/76] Simplify custom_data stuff

---
 freqtrade/persistence/custom_data.py | 57 ++++++++++++++--------------
 freqtrade/persistence/models.py      |  6 +--
 freqtrade/persistence/trade_model.py | 24 ++++++++++--
 3 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index bf6056278..e8fa0d960 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -14,7 +14,7 @@ from freqtrade.util import dt_now
 logger = logging.getLogger(__name__)
 
 
-class CustomData(ModelBase):
+class _CustomData(ModelBase):
     """
     CustomData database model
     Keeps records of metadata as key/value store
@@ -41,6 +41,9 @@ class CustomData(ModelBase):
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, default=dt_now)
     updated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
+    # Empty container value - not persisted, but filled with cd_value on query
+    value: Any = None
+
     def __repr__(self):
         create_time = (self.created_at.strftime(DATETIME_PRINT_FORMAT)
                        if self.created_at is not None else None)
@@ -52,7 +55,7 @@ class CustomData(ModelBase):
 
     @classmethod
     def query_cd(cls, key: Optional[str] = None,
-                 trade_id: Optional[int] = None) -> Sequence['CustomData']:
+                 trade_id: Optional[int] = None) -> Sequence['_CustomData']:
         """
         Get all CustomData, if trade_id is not specified
         return will be for generic values not tied to a trade
@@ -60,11 +63,11 @@ class CustomData(ModelBase):
         """
         filters = []
         if trade_id is not None:
-            filters.append(CustomData.ft_trade_id == trade_id)
+            filters.append(_CustomData.ft_trade_id == trade_id)
         if key is not None:
-            filters.append(CustomData.cd_key.ilike(key))
+            filters.append(_CustomData.cd_key.ilike(key))
 
-        return CustomData.session.scalars(select(CustomData).filter(*filters)).all()
+        return _CustomData.session.scalars(select(_CustomData).filter(*filters)).all()
 
 
 class CustomDataWrapper:
@@ -75,9 +78,15 @@ class CustomDataWrapper:
     """
 
     use_db = True
-    custom_data: List[CustomData] = []
+    custom_data: List[_CustomData] = []
     unserialized_types = ['bool', 'float', 'int', 'str']
 
+    @staticmethod
+    def _convert_custom_data(data: _CustomData) -> _CustomData:
+        if data.cd_type not in CustomDataWrapper.unserialized_types:
+            data.value = json.loads(data.cd_value)
+        return data
+
     @staticmethod
     def reset_custom_data() -> None:
         """
@@ -88,17 +97,15 @@ class CustomDataWrapper:
 
     @staticmethod
     def get_custom_data(key: Optional[str] = None,
-                        trade_id: Optional[int] = None) -> CustomData:
+                        trade_id: Optional[int] = None) -> List[_CustomData]:
         if trade_id is None:
             trade_id = 0
 
         if CustomDataWrapper.use_db:
-            filtered_custom_data = []
-            for data_entry in CustomData.query_cd(trade_id=trade_id, key=key):
-                if data_entry.cd_type not in CustomDataWrapper.unserialized_types:
-                    data_entry.cd_value = json.loads(data_entry.cd_value)
-                filtered_custom_data.append(data_entry)
-            return filtered_custom_data
+            filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter(
+                _CustomData.ft_trade_id == trade_id,
+                _CustomData.cd_key.ilike(key))).all()
+
         else:
             filtered_custom_data = [
                 data_entry for data_entry in CustomDataWrapper.custom_data
@@ -109,19 +116,19 @@ class CustomDataWrapper:
                     data_entry for data_entry in filtered_custom_data
                     if (data_entry.cd_key.casefold() == key.casefold())
                 ]
-            return filtered_custom_data
+        return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data]
 
     @staticmethod
     def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
 
         value_type = type(value).__name__
-        value_db = None
 
         if value_type not in CustomDataWrapper.unserialized_types:
             try:
                 value_db = json.dumps(value)
             except TypeError as e:
                 logger.warning(f"could not serialize {key} value due to {e}")
+                return
         else:
             value_db = str(value)
 
@@ -134,19 +141,19 @@ class CustomDataWrapper:
             data_entry.cd_value = value_db
             data_entry.updated_at = dt_now()
         else:
-            data_entry = CustomData(
+            data_entry = _CustomData(
                 ft_trade_id=trade_id,
                 cd_key=key,
                 cd_type=value_type,
                 cd_value=value_db,
-                created_at=dt_now()
+                created_at=dt_now(),
             )
+        data_entry.value = value
 
         if CustomDataWrapper.use_db and value_db is not None:
-            data_entry.cd_value = value_db
-            CustomData.session.add(data_entry)
-            CustomData.session.commit()
-        elif not CustomDataWrapper.use_db:
+            _CustomData.session.add(data_entry)
+            _CustomData.session.commit()
+        else:
             cd_index = -1
             for index, data_entry in enumerate(CustomDataWrapper.custom_data):
                 if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
@@ -161,11 +168,3 @@ class CustomDataWrapper:
                 CustomDataWrapper.custom_data[cd_index] = data_entry
             else:
                 CustomDataWrapper.custom_data.append(data_entry)
-
-    @staticmethod
-    def get_all_custom_data() -> List[CustomData]:
-
-        if CustomDataWrapper.use_db:
-            return list(CustomData.query_cd())
-        else:
-            return CustomDataWrapper.custom_data
diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py
index 189b80fa6..1a69b271c 100644
--- a/freqtrade/persistence/models.py
+++ b/freqtrade/persistence/models.py
@@ -13,7 +13,7 @@ from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.persistence.base import ModelBase
-from freqtrade.persistence.custom_data import CustomData
+from freqtrade.persistence.custom_data import _CustomData
 from freqtrade.persistence.key_value_store import _KeyValueStoreModel
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
@@ -79,8 +79,8 @@ def init_db(db_url: str) -> None:
     Order.session = Trade.session
     PairLock.session = Trade.session
     _KeyValueStoreModel.session = Trade.session
-    CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True),
-                                        scopefunc=get_request_or_thread_id)
+    _CustomData.session = scoped_session(sessionmaker(bind=engine, autoflush=True),
+                                         scopefunc=get_request_or_thread_id)
 
     previous_tables = inspect(engine).get_table_names()
     ModelBase.metadata.create_all(engine)
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 7487c72b3..55a075cc9 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -23,7 +23,7 @@ from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precisi
 from freqtrade.leverage import interest
 from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.base import ModelBase, SessionType
-from freqtrade.persistence.custom_data import CustomData, CustomDataWrapper
+from freqtrade.persistence.custom_data import CustomDataWrapper, _CustomData
 from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts
 
 
@@ -1206,10 +1206,28 @@ class LocalTrade:
                 ]
 
     def set_custom_data(self, key: str, value: Any) -> None:
+        """
+        Set custom data for this trade
+        :param key: key of the custom data
+        :param value: value of the custom data (must be JSON serializable)
+        """
         CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
 
-    def get_custom_data(self, key: Optional[str]) -> List[_CustomData]:
-        return CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
+    def get_custom_data(self, key: str) -> Optional[_CustomData]:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
+        if data:
+            return data[0]
+        return None
+
+    def get_all_custom_data(self) -> List[_CustomData]:
+        """
+        Get all custom data for this trade
+        """
+        return CustomDataWrapper.get_custom_data(trade_id=self.id)
 
     @property
     def nr_of_successful_entries(self) -> int:

From 790c7e386a3f2146cbee39e812c7f78fd9a551ed Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:27:56 +0100
Subject: [PATCH 28/76] re-adjust logic for custom_data in rpc module

---
 freqtrade/rpc/rpc.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 0c34c19ec..34d33ecde 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -1005,7 +1005,13 @@ class RPC:
         if trade is None:
             return []
         # Query custom_data
-        custom_data = trade.get_custom_data(key=key)
+        custom_data = []
+        if key:
+            data = trade.get_custom_data(key=key)
+            if data:
+                custom_data = [data]
+        else:
+            custom_data = trade.get_all_custom_data()
         return [
             {
                 'id': data_entry.id,

From 8364a704d6c14fc0b468c3dfce7245aeb994a7e0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:31:44 +0100
Subject: [PATCH 29/76] Fix a few sql gotchas

---
 freqtrade/persistence/custom_data.py | 1 +
 freqtrade/persistence/trade_model.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index e8fa0d960..bf72510be 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -24,6 +24,7 @@ class _CustomData(ModelBase):
       - One metadata entry can only be associated with one Trade
     """
     __tablename__ = 'trade_custom_data'
+    __allow_unmapped__ = True
     session: ClassVar[SessionType]
 
     # Uniqueness should be ensured over pair, order_id
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 55a075cc9..1aa7cb607 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1485,7 +1485,7 @@ class Trade(ModelBase, LocalTrade):
         "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
         innerjoin=True)  # type: ignore
     custom_data: Mapped[List[_CustomData]] = relationship(
-        "CustomData", order_by="CustomData.id", cascade="all, delete-orphan",
+        "_CustomData", cascade="all, delete-orphan",
         lazy="raise")  # type: ignore
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore

From 67b910835e05b76c170f406895a0c66a784742c8 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:39:29 +0100
Subject: [PATCH 30/76] Simplify access to custom_data - users will usually
 only care about the value, not about the metadata.

---
 docs/strategy-advanced.md            | 17 +++++++++++++----
 freqtrade/persistence/trade_model.py | 12 +++++++++++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 69d4ced34..9f0b3c112 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -65,7 +65,7 @@ class AwesomeStrategy(IStrategy):
         for trade in Trade.get_open_order_trades():
             fills = trade.select_filled_orders(trade.entry_side)
             if trade.pair == 'ETH/USDT':
-                trade_entry_type = trade.get_custom_data(key='entry_type').kv_value
+                trade_entry_type = trade.get_custom_data(key='entry_type')
                 if trade_entry_type is None:
                     trade_entry_type = 'breakout' if 'entry_1' in trade.enter_tag else 'dip'
                 elif fills > 1:
@@ -87,7 +87,7 @@ class AwesomeStrategy(IStrategy):
             dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
             current_candle = dataframe.iloc[-1].squeeze()
             # store information about entry adjustment
-            existing_count = trade.get_custom_data(key='num_entry_adjustments').kv_value
+            existing_count = trade.get_custom_data('num_entry_adjustments', default=0)
             if not existing_count:
                 existing_count = 1
             else:
@@ -102,8 +102,8 @@ class AwesomeStrategy(IStrategy):
 
     def custom_exit(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs):
 
-        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments').kv_value
-        trade_entry_type = trade.get_custom_data(key='entry_type').kv_value
+        entry_adjustment_count = trade.get_custom_data(key='num_entry_adjustments')
+        trade_entry_type = trade.get_custom_data(key='entry_type')
         if entry_adjustment_count is None:
             if current_profit > 0.01 and (current_time - timedelta(minutes=100) > trade.open_date_utc):
                 return True, 'exit_1'
@@ -123,6 +123,15 @@ class AwesomeStrategy(IStrategy):
 !!! Warning "Non-serializable data"
     If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.
 
+??? Note "All attributes"
+    custom-data has the following accessors through the Trade object (assumed as `trade` below):
+
+    * `trade.get_custom_data(key='something', default=0)` - Returns the actual value given in the type provided.
+    * `trade.get_custom_data_entry(key='something')` - Returns the entry - including metadata. The value is accessible via `.value` property.
+    * `trade.set_custom_data(key='something', value={'some': 'value'})` - set or update the corresponding key for this trade. Value must be serializable - and we recommend to keep the stored data relatively small.
+
+    "value" can be any type (both in setting and receiving) - but must be json serializable.
+
 ## Dataframe access
 
 You may access dataframe in various strategy functions by querying it from dataprovider.
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 1aa7cb607..abbc69f75 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1213,7 +1213,17 @@ class LocalTrade:
         """
         CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
 
-    def get_custom_data(self, key: str) -> Optional[_CustomData]:
+    def get_custom_data(self, key: str, default: Any = None) -> Any:
+        """
+        Get custom data for this trade
+        :param key: key of the custom data
+        """
+        data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
+        if data:
+            return data[0]
+        return default
+
+    def get_custom_data_entry(self, key: str) -> Optional[_CustomData]:
         """
         Get custom data for this trade
         :param key: key of the custom data

From 6a6e3aacf33da282006d56d775e301ee33d8274b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 12 Feb 2024 20:51:44 +0100
Subject: [PATCH 31/76] Fix broken deletion

---
 freqtrade/persistence/custom_data.py | 4 ++++
 freqtrade/persistence/trade_model.py | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index bf72510be..0eb14738c 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -96,6 +96,10 @@ class CustomDataWrapper:
         if not CustomDataWrapper.use_db:
             CustomDataWrapper.custom_data = []
 
+    @staticmethod
+    def delete_custom_data(trade_id: int) -> None:
+        _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete()
+
     @staticmethod
     def get_custom_data(key: Optional[str] = None,
                         trade_id: Optional[int] = None) -> List[_CustomData]:
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index abbc69f75..a12a842c3 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1600,8 +1600,7 @@ class Trade(ModelBase, LocalTrade):
         for order in self.orders:
             Order.session.delete(order)
 
-        for entry in self.custom_data:
-            _CustomData.session.delete(entry)
+        CustomDataWrapper.delete_custom_data(trade_id=self.id)
 
         _CustomData.session.commit()
         Trade.session.delete(self)

From 304f52ab79c899a9b6f6cfeed95f125efcac0c89 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 13 Feb 2024 07:06:32 +0100
Subject: [PATCH 32/76] Fix some minor custom-data bugs

---
 freqtrade/persistence/custom_data.py | 27 +++++++++++++++++++--------
 freqtrade/persistence/trade_model.py |  8 ++++----
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index 0eb14738c..f5d6587bf 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -32,7 +32,7 @@ class _CustomData(ModelBase):
     __table_args__ = (UniqueConstraint('ft_trade_id', 'cd_key', name="_trade_id_cd_key"),)
 
     id = mapped_column(Integer, primary_key=True)
-    ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True, default=0)
+    ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True)
 
     trade = relationship("Trade", back_populates="custom_data")
 
@@ -84,7 +84,15 @@ class CustomDataWrapper:
 
     @staticmethod
     def _convert_custom_data(data: _CustomData) -> _CustomData:
-        if data.cd_type not in CustomDataWrapper.unserialized_types:
+        if data.cd_type in CustomDataWrapper.unserialized_types:
+            data.value = data.cd_value
+            if data.cd_type == 'bool':
+                data.value = data.cd_value.lower() == 'true'
+            elif data.cd_type == 'int':
+                data.value = int(data.cd_value)
+            elif data.cd_type == 'float':
+                data.value = float(data.cd_value)
+        else:
             data.value = json.loads(data.cd_value)
         return data
 
@@ -101,15 +109,18 @@ class CustomDataWrapper:
         _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete()
 
     @staticmethod
-    def get_custom_data(key: Optional[str] = None,
-                        trade_id: Optional[int] = None) -> List[_CustomData]:
+    def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]:
         if trade_id is None:
             trade_id = 0
 
         if CustomDataWrapper.use_db:
-            filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter(
+            filters = [
                 _CustomData.ft_trade_id == trade_id,
-                _CustomData.cd_key.ilike(key))).all()
+            ]
+            if key is not None:
+                filters.append(_CustomData.cd_key.ilike(key))
+            filtered_custom_data = _CustomData.session.scalars(select(_CustomData).filter(
+                *filters)).all()
 
         else:
             filtered_custom_data = [
@@ -124,7 +135,7 @@ class CustomDataWrapper:
         return [CustomDataWrapper._convert_custom_data(d) for d in filtered_custom_data]
 
     @staticmethod
-    def set_custom_data(key: str, value: Any, trade_id: Optional[int] = None) -> None:
+    def set_custom_data(trade_id: int, key: str, value: Any) -> None:
 
         value_type = type(value).__name__
 
@@ -140,7 +151,7 @@ class CustomDataWrapper:
         if trade_id is None:
             trade_id = 0
 
-        custom_data = CustomDataWrapper.get_custom_data(key=key, trade_id=trade_id)
+        custom_data = CustomDataWrapper.get_custom_data(trade_id=trade_id, key=key)
         if custom_data:
             data_entry = custom_data[0]
             data_entry.cd_value = value_db
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index a12a842c3..d2a92a554 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -1211,16 +1211,16 @@ class LocalTrade:
         :param key: key of the custom data
         :param value: value of the custom data (must be JSON serializable)
         """
-        CustomDataWrapper.set_custom_data(key=key, value=value, trade_id=self.id)
+        CustomDataWrapper.set_custom_data(trade_id=self.id, key=key, value=value)
 
     def get_custom_data(self, key: str, default: Any = None) -> Any:
         """
         Get custom data for this trade
         :param key: key of the custom data
         """
-        data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
         if data:
-            return data[0]
+            return data[0].value
         return default
 
     def get_custom_data_entry(self, key: str) -> Optional[_CustomData]:
@@ -1228,7 +1228,7 @@ class LocalTrade:
         Get custom data for this trade
         :param key: key of the custom data
         """
-        data = CustomDataWrapper.get_custom_data(key=key, trade_id=self.id)
+        data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
         if data:
             return data[0]
         return None

From 9be7759e423f9e4d5382c3176225aae8b20a1d8f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 13 Feb 2024 07:10:11 +0100
Subject: [PATCH 33/76] Add initial test for custom_data

---
 tests/persistence/test_trade_custom_data.py | 34 +++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 tests/persistence/test_trade_custom_data.py

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
new file mode 100644
index 000000000..d411a898f
--- /dev/null
+++ b/tests/persistence/test_trade_custom_data.py
@@ -0,0 +1,34 @@
+import pytest
+from sqlalchemy import select
+
+from freqtrade.persistence import Trade
+from tests.conftest import create_mock_trades_usdt
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_trade_custom_data(fee):
+    create_mock_trades_usdt(fee)
+
+    trade1 = Trade.session.scalars(select(Trade)).first()
+
+    assert trade1.get_all_custom_data() == []
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_float', 1.55)
+    trade1.set_custom_data('test_bool', True)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+
+    assert trade1.get_custom_data('test_str') == 'test_value'
+
+    assert trade1.get_custom_data('test_int') == 1
+    assert isinstance(trade1.get_custom_data('test_int'), int)
+
+    assert trade1.get_custom_data('test_float') == 1.55
+    assert isinstance(trade1.get_custom_data('test_float'), float)
+
+    assert trade1.get_custom_data('test_bool') is True
+    assert isinstance(trade1.get_custom_data('test_bool'), bool)
+
+    assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
+    assert isinstance(trade1.get_custom_data('test_dict'), dict)
+    assert len(trade1.get_all_custom_data()) == 5

From d5b21f2a32e9acc8460ea32b9030b2b67288a4bd Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 13 Feb 2024 07:16:09 +0100
Subject: [PATCH 34/76] Fix bug in backtest mode

---
 freqtrade/persistence/custom_data.py | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index f5d6587bf..3ebcd0f48 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -110,8 +110,6 @@ class CustomDataWrapper:
 
     @staticmethod
     def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]:
-        if trade_id is None:
-            trade_id = 0
 
         if CustomDataWrapper.use_db:
             filters = [
@@ -170,17 +168,6 @@ class CustomDataWrapper:
             _CustomData.session.add(data_entry)
             _CustomData.session.commit()
         else:
-            cd_index = -1
-            for index, data_entry in enumerate(CustomDataWrapper.custom_data):
-                if data_entry.ft_trade_id == trade_id and data_entry.cd_key == key:
-                    cd_index = index
-                    break
-
-            if cd_index >= 0:
-                data_entry.cd_type = value_type
-                data_entry.cd_value = value_db
-                data_entry.updated_at = dt_now()
-
-                CustomDataWrapper.custom_data[cd_index] = data_entry
-            else:
+            if not custom_data:
                 CustomDataWrapper.custom_data.append(data_entry)
+            # Existing data will have updated interactively.

From ab062d7bb145f163b78739ee69f2f1228879e3ee Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 13 Feb 2024 07:16:20 +0100
Subject: [PATCH 35/76] Add test to run in backtest mode

---
 tests/persistence/test_trade_custom_data.py | 23 +++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
index d411a898f..12767b811 100644
--- a/tests/persistence/test_trade_custom_data.py
+++ b/tests/persistence/test_trade_custom_data.py
@@ -1,15 +1,23 @@
 import pytest
-from sqlalchemy import select
 
-from freqtrade.persistence import Trade
+from freqtrade.persistence import Trade, disable_database_use, enable_database_use
+from freqtrade.persistence.custom_data import CustomDataWrapper
 from tests.conftest import create_mock_trades_usdt
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_trade_custom_data(fee):
-    create_mock_trades_usdt(fee)
+@pytest.mark.parametrize("use_db", [True, False])
+def test_trade_custom_data(fee, use_db):
+    if not use_db:
+        disable_database_use('5m')
+    Trade.reset_trades()
+    CustomDataWrapper.reset_custom_data()
 
-    trade1 = Trade.session.scalars(select(Trade)).first()
+    create_mock_trades_usdt(fee, use_db=use_db)
+
+    trade1 = Trade.get_trades_proxy()[0]
+    if not use_db:
+        trade1.id = 1
 
     assert trade1.get_all_custom_data() == []
     trade1.set_custom_data('test_str', 'test_value')
@@ -18,7 +26,10 @@ def test_trade_custom_data(fee):
     trade1.set_custom_data('test_bool', True)
     trade1.set_custom_data('test_dict', {'test': 'dict'})
 
+    assert len(trade1.get_all_custom_data()) == 5
     assert trade1.get_custom_data('test_str') == 'test_value'
+    trade1.set_custom_data('test_str', 'test_value_updated')
+    assert trade1.get_custom_data('test_str') == 'test_value_updated'
 
     assert trade1.get_custom_data('test_int') == 1
     assert isinstance(trade1.get_custom_data('test_int'), int)
@@ -31,4 +42,4 @@ def test_trade_custom_data(fee):
 
     assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
     assert isinstance(trade1.get_custom_data('test_dict'), dict)
-    assert len(trade1.get_all_custom_data()) == 5
+    enable_database_use()

From e8ca9ce39b23172baa4ecc8e954b1a5858eed25d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 20 Feb 2024 20:00:45 +0100
Subject: [PATCH 36/76] Add testconfirming correct functioning

---
 tests/persistence/test_trade_custom_data.py | 40 ++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
index 12767b811..cc86342d2 100644
--- a/tests/persistence/test_trade_custom_data.py
+++ b/tests/persistence/test_trade_custom_data.py
@@ -2,7 +2,7 @@ import pytest
 
 from freqtrade.persistence import Trade, disable_database_use, enable_database_use
 from freqtrade.persistence.custom_data import CustomDataWrapper
-from tests.conftest import create_mock_trades_usdt
+from tests.conftest import EXMS, create_mock_trades_usdt, get_patched_freqtradebot
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -43,3 +43,41 @@ def test_trade_custom_data(fee, use_db):
     assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
     assert isinstance(trade1.get_custom_data('test_dict'), dict)
     enable_database_use()
+
+
+def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_rate', return_value=0.50)
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None)
+    default_conf_usdt["minimal_roi"] = {
+        "0":  100
+    }
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    create_mock_trades_usdt(fee)
+
+    trade1 = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+
+    def custom_exit(pair, trade, **kwargs):
+
+        if pair == 'ADA/USDT':
+            custom_val = trade.get_custom_data('test_str')
+            custom_val_i = trade.get_custom_data('test_int')
+
+            return f"{custom_val}_{custom_val_i}"
+
+    freqtrade.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(freqtrade.strategy, 'custom_exit')
+    trades = Trade.get_open_trades()
+    freqtrade.exit_positions(trades)
+    Trade.commit()
+
+    trade_after = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    assert trade_after.get_custom_data('test_str') == 'test_value'
+    assert trade_after.get_custom_data('test_int') == 1
+    # 2 open pairs eligible for exit
+    assert ff_spy.call_count == 2
+
+    assert trade_after.exit_reason == 'test_value_1'

From c511d65d2e10cc858591c36c2a1fc99673538eec Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Tue, 20 Feb 2024 20:25:12 +0100
Subject: [PATCH 37/76] Add backtesting test

---
 tests/persistence/test_trade_custom_data.py | 83 ++++++++++++++++++++-
 1 file changed, 79 insertions(+), 4 deletions(-)

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
index cc86342d2..fe73f5f6b 100644
--- a/tests/persistence/test_trade_custom_data.py
+++ b/tests/persistence/test_trade_custom_data.py
@@ -1,8 +1,14 @@
+from copy import deepcopy
+from unittest.mock import MagicMock
+
 import pytest
 
+from freqtrade.data.history.history_utils import get_timerange
+from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import Trade, disable_database_use, enable_database_use
 from freqtrade.persistence.custom_data import CustomDataWrapper
-from tests.conftest import EXMS, create_mock_trades_usdt, get_patched_freqtradebot
+from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data,
+                            get_patched_freqtradebot, patch_exchange)
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -49,9 +55,7 @@ def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
 
     mocker.patch(f'{EXMS}.get_rate', return_value=0.50)
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None)
-    default_conf_usdt["minimal_roi"] = {
-        "0":  100
-    }
+    default_conf_usdt["minimal_roi"] = {"0":  100}
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
     create_mock_trades_usdt(fee)
@@ -81,3 +85,74 @@ def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
     assert ff_spy.call_count == 2
 
     assert trade_after.exit_reason == 'test_value_1'
+
+
+def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_fee', fee)
+    mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10)
+    mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf'))
+    mocker.patch(f"{EXMS}.get_max_leverage", return_value=10)
+    mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting._run_funding_fees')
+
+    patch_exchange(mocker)
+    default_conf_usdt.update({
+        "stake_amount": 100.0,
+        "max_open_trades": 2,
+        "dry_run_wallet": 1000.0,
+        "strategy": "StrategyTestV3",
+        "trading_mode": "futures",
+        "margin_mode": "isolated",
+        "stoploss": -2,
+        "minimal_roi": {"0":  100},
+    })
+    default_conf_usdt['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}]
+    backtesting = Backtesting(default_conf_usdt)
+
+    df = generate_test_data(default_conf_usdt['timeframe'], 100, '2022-01-01 00:00:00+00:00')
+
+    pair_exp = 'XRP/USDT:USDT'
+
+    def custom_exit(pair, trade, **kwargs):
+        custom_val = trade.get_custom_data('test_str')
+        custom_val_i = trade.get_custom_data('test_int', 0)
+
+        if pair == pair_exp:
+            trade.set_custom_data('test_str', 'test_value')
+            trade.set_custom_data('test_int', custom_val_i + 1)
+
+        if custom_val_i >= 2:
+            return f"{custom_val}_{custom_val_i}"
+
+    backtesting._set_strategy(backtesting.strategylist[0])
+    processed = backtesting.strategy.advise_all_indicators({
+        pair_exp: df,
+        'BTC/USDT:USDT': df,
+    })
+
+    def fun(dataframe, *args, **kwargs):
+        dataframe.loc[dataframe.index == 50, 'enter_long'] = 1
+        return dataframe
+
+    backtesting.strategy.advise_entry = fun
+    backtesting.strategy.leverage = MagicMock(return_value=1)
+    backtesting.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(backtesting.strategy, 'custom_exit')
+
+    min_date, max_date = get_timerange(processed)
+
+    result = backtesting.backtest(
+        processed=deepcopy(processed),
+        start_date=min_date,
+        end_date=max_date,
+    )
+    results = result['results']
+    assert not results.empty
+    assert len(results) == 2
+    assert results['pair'][0] == pair_exp
+    assert results['pair'][1] == 'BTC/USDT:USDT'
+    assert results['exit_reason'][0] == 'test_value_2'
+    assert results['exit_reason'][1] == 'exit_signal'
+
+    assert ff_spy.call_count == 7

From 4bbb3174b21771f2336f64ccf73c54dec626460d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 21 Feb 2024 06:52:22 +0100
Subject: [PATCH 38/76] re-enable use_database after bt test

---
 tests/persistence/test_trade_custom_data.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
index fe73f5f6b..bf1d73e1d 100644
--- a/tests/persistence/test_trade_custom_data.py
+++ b/tests/persistence/test_trade_custom_data.py
@@ -156,3 +156,4 @@ def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, f
     assert results['exit_reason'][1] == 'exit_signal'
 
     assert ff_spy.call_count == 7
+    enable_database_use()

From c7fff45bef14cb8c3009378aa141cf2570334e3f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Thu, 22 Feb 2024 19:01:33 +0100
Subject: [PATCH 39/76] Fix test leakage

---
 tests/persistence/test_trade_custom_data.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/persistence/test_trade_custom_data.py b/tests/persistence/test_trade_custom_data.py
index bf1d73e1d..15241aa93 100644
--- a/tests/persistence/test_trade_custom_data.py
+++ b/tests/persistence/test_trade_custom_data.py
@@ -48,7 +48,8 @@ def test_trade_custom_data(fee, use_db):
 
     assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
     assert isinstance(trade1.get_custom_data('test_dict'), dict)
-    enable_database_use()
+    if not use_db:
+        enable_database_use()
 
 
 def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
@@ -156,4 +157,4 @@ def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, f
     assert results['exit_reason'][1] == 'exit_signal'
 
     assert ff_spy.call_count == 7
-    enable_database_use()
+    Backtesting.cleanup()

From 6307e1630498e015e8df03fdc21a55e583160414 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 23 Feb 2024 06:45:28 +0100
Subject: [PATCH 40/76] Properly format notification date

---
 freqtrade/rpc/telegram.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index d0e12cc4a..4f4ea17d3 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -33,7 +33,7 @@ from freqtrade.misc import chunks, plural
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException, RPCHandler
 from freqtrade.rpc.rpc_types import RPCEntryMsg, RPCExitMsg, RPCOrderMsg, RPCSendMsg
-from freqtrade.util import dt_humanize, fmt_coin, round_value
+from freqtrade.util import dt_humanize, fmt_coin, format_date, round_value
 
 
 MAX_MESSAGE_LENGTH = MessageLimit.MAX_TEXT_LENGTH
@@ -1797,8 +1797,8 @@ class Telegram(RPCHandler):
                         f"*Trade ID:* `{result['ft_trade_id']}`",
                         f"*Type:* `{result['cd_type']}`",
                         f"*Value:* `{result['cd_value']}`",
-                        f"*Create Date:* `{result['created_at']}`",
-                        f"*Update Date:* `{result['updated_at']}`"
+                        f"*Create Date:* `{format_date(result['created_at'])}`",
+                        f"*Update Date:* `{format_date(result['updated_at'])}`"
                     ]
                     # Filter empty lines using list-comprehension
                     messages.append("\n".join([line for line in lines if line]))

From 1176c16b93bd8b4d8375236b7a9011880b9f306c Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 12:41:51 +0100
Subject: [PATCH 41/76] REmove unnecessary assignment

---
 freqtrade/optimize/backtesting.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
index 4b217bcf6..7147ad14f 100644
--- a/freqtrade/optimize/backtesting.py
+++ b/freqtrade/optimize/backtesting.py
@@ -337,7 +337,6 @@ class Backtesting:
         self.disable_database_use()
         PairLocks.reset_locks()
         Trade.reset_trades()
-        CustomDataWrapper.use_db = False
         CustomDataWrapper.reset_custom_data()
         self.rejected_trades = 0
         self.timedout_entry_orders = 0

From 30b4f271522f233fe50ecd9e8b67e6120446d581 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 12:46:46 +0100
Subject: [PATCH 42/76] Cleanup some nitpicks

---
 freqtrade/persistence/custom_data.py  | 1 +
 freqtrade/persistence/trade_model.py  | 4 +---
 tests/persistence/test_persistence.py | 1 +
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/freqtrade/persistence/custom_data.py b/freqtrade/persistence/custom_data.py
index 3ebcd0f48..81a9e7ad6 100644
--- a/freqtrade/persistence/custom_data.py
+++ b/freqtrade/persistence/custom_data.py
@@ -107,6 +107,7 @@ class CustomDataWrapper:
     @staticmethod
     def delete_custom_data(trade_id: int) -> None:
         _CustomData.session.query(_CustomData).filter(_CustomData.ft_trade_id == trade_id).delete()
+        _CustomData.session.commit()
 
     @staticmethod
     def get_custom_data(*, trade_id: int, key: Optional[str] = None) -> List[_CustomData]:
diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py
index 06a6e818d..e74bc1f48 100644
--- a/freqtrade/persistence/trade_model.py
+++ b/freqtrade/persistence/trade_model.py
@@ -343,7 +343,6 @@ class LocalTrade:
     id: int = 0
 
     orders: List[Order] = []
-    custom_data: List[_CustomData] = []
 
     exchange: str = ''
     pair: str = ''
@@ -1507,7 +1506,7 @@ class Trade(ModelBase, LocalTrade):
         innerjoin=True)  # type: ignore
     custom_data: Mapped[List[_CustomData]] = relationship(
         "_CustomData", cascade="all, delete-orphan",
-        lazy="raise")  # type: ignore
+        lazy="raise")
 
     exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
     pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
@@ -1613,7 +1612,6 @@ class Trade(ModelBase, LocalTrade):
 
         CustomDataWrapper.delete_custom_data(trade_id=self.id)
 
-        _CustomData.session.commit()
         Trade.session.delete(self)
         Trade.commit()
 
diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py
index 0e0e70ee8..18f28da2b 100644
--- a/tests/persistence/test_persistence.py
+++ b/tests/persistence/test_persistence.py
@@ -2099,6 +2099,7 @@ def test_Trade_object_idem():
         'get_mix_tag_performance',
         'get_trading_volume',
         'validate_string_len',
+        'custom_data'
     )
     EXCLUDES2 = ('trades', 'trades_open', 'bt_trades_open_pp', 'bt_open_open_trade_count',
                  'total_profit', 'from_json',)

From c1ae110080e4b966efe1c5adf60b013b1b6a2e5b Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 12:56:42 +0100
Subject: [PATCH 43/76] Improve documentation

---
 docs/strategy-advanced.md | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 9f0b3c112..36185676c 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -14,10 +14,10 @@ The call sequence of the methods described here is covered under [bot execution
 ## Storing information (Non-Persistent)
 
 !!! Warning "Deprecated"
-    This method of storing information is deprecated, and we do advise against using non-persistent storage.  
+    This method of storing information is deprecated and we do advise against using non-persistent storage.  
     Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead.
 
-    It's content has therefore be collapsed.
+    It's content has therefore been collapsed.
 
 ??? Abstract "Storing information"
     Storing information can be accomplished by creating a new dictionary within the strategy class.
@@ -49,11 +49,12 @@ The call sequence of the methods described here is covered under [bot execution
 
 ## Storing information (Persistent)
 
-Storing information can also be performed in a persistent manner. Freqtrade allows storing/retrieving user custom information associated with a specific trade.
+Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database.
 
-Using a trade object, information can be stored using `trade_obj.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade_obj.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
+Using a trade object, information can be stored using `trade.set_custom_data(key='my_key', value=my_value)` and retrieved using `trade.get_custom_data(key='my_key')`. Each data entry is associated with a trade and a user supplied key (of type `string`). This means that this can only be used in callbacks that also provide a trade object.
 
-For the data to be able to be stored within the database it must be serialized. This is done by converting it to a JSON formatted string.
+For the data to be able to be stored within the database, freqtrade must serialized the data. This is done by converting the data to a JSON formatted string.
+Freqtrade will attempt to reverse this action on retrieval, so from a strategy perspective, this should not be relevant.
 
 ```python
 from freqtrade.persistence import Trade
@@ -116,9 +117,11 @@ class AwesomeStrategy(IStrategy):
         return False, None
 ```
 
+The above is a simple example - there are simpler ways to retrieve trade data like entry-adjustments.
+
 !!! Note
     It is recommended that simple data types are used `[bool, int, float, str]` to ensure no issues when serializing the data that needs to be stored.
-    Storing big junks of data may lead to unintended side-effects, like a database becoming big pretty fast (and as a consequence, also slow).
+    Storing big junks of data may lead to unintended side-effects, like a database becoming big (and as a consequence, also slow).
 
 !!! Warning "Non-serializable data"
     If supplied data cannot be serialized a warning is logged and the entry for the specified `key` will contain `None` as data.

From ceb461a25285d9a0ce7fc44c6ceac0b1cccb3552 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 12:58:10 +0100
Subject: [PATCH 44/76] Switch sequence of information documentation

---
 docs/strategy-advanced.md | 72 +++++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md
index 36185676c..debd5bc1b 100644
--- a/docs/strategy-advanced.md
+++ b/docs/strategy-advanced.md
@@ -11,42 +11,6 @@ The call sequence of the methods described here is covered under [bot execution
 !!! Tip
     Start off with a strategy template containing all available callback methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced`
 
-## Storing information (Non-Persistent)
-
-!!! Warning "Deprecated"
-    This method of storing information is deprecated and we do advise against using non-persistent storage.  
-    Please use the below [Persistent Storing Information Section](#storing-information-persistent) instead.
-
-    It's content has therefore been collapsed.
-
-??? Abstract "Storing information"
-    Storing information can be accomplished by creating a new dictionary within the strategy class.
-
-    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
-
-    ```python
-    class AwesomeStrategy(IStrategy):
-        # Create custom dictionary
-        custom_info = {}
-
-        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-            # Check if the entry already exists
-            if not metadata["pair"] in self.custom_info:
-                # Create empty entry for this pair
-                self.custom_info[metadata["pair"]] = {}
-
-            if "crosstime" in self.custom_info[metadata["pair"]]:
-                self.custom_info[metadata["pair"]]["crosstime"] += 1
-            else:
-                self.custom_info[metadata["pair"]]["crosstime"] = 1
-    ```
-
-    !!! Warning
-        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
-
-    !!! Note
-        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
-
 ## Storing information (Persistent)
 
 Freqtrade allows storing/retrieving user custom information associated with a specific trade in the database.
@@ -135,6 +99,42 @@ The above is a simple example - there are simpler ways to retrieve trade data li
 
     "value" can be any type (both in setting and receiving) - but must be json serializable.
 
+## Storing information (Non-Persistent)
+
+!!! Warning "Deprecated"
+    This method of storing information is deprecated and we do advise against using non-persistent storage.  
+    Please use [Persistent Storage](#storing-information-persistent) instead.
+
+    It's content has therefore been collapsed.
+
+??? Abstract "Storing information"
+    Storing information can be accomplished by creating a new dictionary within the strategy class.
+
+    The name of the variable can be chosen at will, but should be prefixed with `custom_` to avoid naming collisions with predefined strategy variables.
+
+    ```python
+    class AwesomeStrategy(IStrategy):
+        # Create custom dictionary
+        custom_info = {}
+
+        def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Check if the entry already exists
+            if not metadata["pair"] in self.custom_info:
+                # Create empty entry for this pair
+                self.custom_info[metadata["pair"]] = {}
+
+            if "crosstime" in self.custom_info[metadata["pair"]]:
+                self.custom_info[metadata["pair"]]["crosstime"] += 1
+            else:
+                self.custom_info[metadata["pair"]]["crosstime"] = 1
+    ```
+
+    !!! Warning
+        The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
+
+    !!! Note
+        If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
+
 ## Dataframe access
 
 You may access dataframe in various strategy functions by querying it from dataprovider.

From 265a7123dad8a68a8df6d0e258fc41bdc9d81211 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 13:10:50 +0100
Subject: [PATCH 45/76] Add explicit test for telegram functionality of
 list-custom-data

---
 tests/rpc/test_rpc_telegram.py | 46 ++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py
index 29a2b2723..3bd372b19 100644
--- a/tests/rpc/test_rpc_telegram.py
+++ b/tests/rpc/test_rpc_telegram.py
@@ -2657,3 +2657,49 @@ async def test_change_market_direction(default_conf, mocker, update) -> None:
     context.args = ["invalid"]
     await telegram._changemarketdir(update, context)
     assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.LONG
+
+
+async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee, mocker) -> None:
+
+    mocker.patch.multiple(
+        EXMS,
+        fetch_ticker=ticker,
+        get_fee=fee,
+    )
+    telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt)
+
+    # Create some test data
+    create_mock_trades_usdt(fee)
+    # No trade id
+    context = MagicMock()
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert 'Trade-id not set.' in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
+    #
+    context.args = ['1']
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert (
+        "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0]
+    )
+    msg_mock.reset_mock()
+
+    # Add some custom data
+    trade1 = Trade.get_trades_proxy()[0]
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+    Trade.commit()
+    context.args = [f"{trade1.id}"]
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 3
+    assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0]
+    assert (
+        "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n"
+        "*Value:* `1`\n*Create Date:*") in msg_mock.call_args_list[1][0][0]
+    assert (
+        '*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n'
+        '*Value:* `{"test": "dict"}`\n*Create Date:* `') in msg_mock.call_args_list[2][0][0]
+
+    msg_mock.reset_mock()

From 255ea88638957d19fcd439987ae32d383243f810 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 3 Mar 2024 15:24:26 +0100
Subject: [PATCH 46/76] Add to telegram documentation

---
 docs/telegram-usage.md    | 1 +
 freqtrade/rpc/telegram.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md
index e4dc02c76..2709baf9a 100644
--- a/docs/telegram-usage.md
+++ b/docs/telegram-usage.md
@@ -181,6 +181,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/locks` | Show currently locked pairs.
 | `/unlock <pair or lock_id>` | Remove the lock for this pair (or for this lock id).
 | `/marketdir [long | short | even | none]` | Updates the user managed variable that represents the current market direction. If no direction is provided, the currently set direction will be displayed.
+| `/list_custom_data <trade_id> [key]` | List custom_data for Trade ID & Key combination. If no Key is supplied it will list all key-value pairs found for that Trade ID.
 | **Modify Trade states** |
 | `/forceexit <trade_id> | /fx <tradeid>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
 | `/forceexit all | /fx all` | Instantly exits all open trades (Ignoring `minimum_roi`).
diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 4f4ea17d3..f7e7362ef 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1668,6 +1668,8 @@ class Telegram(RPCHandler):
             "*/marketdir [long | short | even | none]:* `Updates the user managed variable "
             "that represents the current market direction. If no direction is provided `"
             "`the currently set market direction will be output.` \n"
+            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
+            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
 
             "_Statistics_\n"
             "------------\n"
@@ -1691,8 +1693,6 @@ class Telegram(RPCHandler):
             "Avg. holding durations for buys and sells.`\n"
             "*/help:* `This help message`\n"
             "*/version:* `Show version`\n"
-            "*/list_custom_data <trade_id> <key>:* `List custom_data for Trade ID & Key combo.`\n"
-            "`If no Key is supplied it will list all key-value pairs found for that Trade ID.`"
             )
 
         await self._send_msg(message, parse_mode=ParseMode.MARKDOWN)

From 510863f939b30152f5f77292cc95f65f179395ad Mon Sep 17 00:00:00 2001
From: xmatthias <5024695+xmatthias@users.noreply.github.com>
Date: Tue, 5 Mar 2024 03:03:38 +0000
Subject: [PATCH 47/76] chore: update pre-commit hooks

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 842c87976..23fa54326 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.2.2'
+    rev: 'v0.3.0'
     hooks:
       - id: ruff
 

From 3e6e534e76ac313f5d22d828bca88097cdb11f6d Mon Sep 17 00:00:00 2001
From: cuinix <915115094@qq.com>
Date: Thu, 7 Mar 2024 13:57:25 +0800
Subject: [PATCH 48/76] fix some typos in docs

Signed-off-by: cuinix <915115094@qq.com>
---
 docs/advanced-backtesting.md          | 4 ++--
 docs/freqai-parameter-table.md        | 2 +-
 docs/freqai-reinforcement-learning.md | 2 +-
 docs/telegram-usage.md                | 2 +-
 docs/webhook-config.md                | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/advanced-backtesting.md b/docs/advanced-backtesting.md
index 3926fb5b1..e91842d64 100644
--- a/docs/advanced-backtesting.md
+++ b/docs/advanced-backtesting.md
@@ -109,12 +109,12 @@ automatically accessible by including them on the indicator-list, and these incl
 - **open_date     :** trade open datetime
 - **close_date    :** trade close datetime
 - **min_rate      :** minimum price seen throughout the position
-- **max_rate      :** maxiumum price seen throughout the position
+- **max_rate      :** maximum price seen throughout the position
 - **open          :** signal candle open price
 - **close         :** signal candle close price
 - **high          :** signal candle high price
 - **low           :** signal candle low price
-- **volume        :** signal candle volumne
+- **volume        :** signal candle volume
 - **profit_ratio  :** trade profit ratio
 - **profit_abs    :** absolute profit return of the trade 
 
diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md
index 905ea479a..055b7b45d 100644
--- a/docs/freqai-parameter-table.md
+++ b/docs/freqai-parameter-table.md
@@ -75,7 +75,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `rl_config` | A dictionary containing the control parameters for a Reinforcement Learning model. <br> **Datatype:** Dictionary.
 | `train_cycles` | Training time steps will be set based on the `train_cycles * number of training data points. <br> **Datatype:** Integer.
 | `max_trade_duration_candles`| Guides the agent training to keep trades below desired length. Example usage shown in `prediction_models/ReinforcementLearner.py` within the customizable `calculate_reward()` function. <br> **Datatype:** int.
-| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentaiton. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
+| `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentation. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
 | `policy_type` | One of the available policy types from stable_baselines3 <br> **Datatype:** string.
 | `max_training_drawdown_pct` | The maximum drawdown that the agent is allowed to experience during training. <br> **Datatype:** float. <br> Default: 0.8
 | `cpu_count` | Number of threads/cpus to dedicate to the Reinforcement Learning training process (depending on if `ReinforcementLearning_multiproc` is selected or not). Recommended to leave this untouched, by default, this value is set to the total number of physical cores minus 1. <br> **Datatype:** int. 
diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md
index c5cda3bc3..3b75e6b71 100644
--- a/docs/freqai-reinforcement-learning.md
+++ b/docs/freqai-reinforcement-learning.md
@@ -142,7 +142,7 @@ Parameter details can be found [here](freqai-parameter-table.md), but in general
 As you begin to modify the strategy and the prediction model, you will quickly realize some important differences between the Reinforcement Learner and the Regressors/Classifiers. Firstly, the strategy does not set a target value (no labels!). Instead, you set the `calculate_reward()` function inside the `MyRLEnv` class (see below). A default `calculate_reward()` is provided inside `prediction_models/ReinforcementLearner.py` to demonstrate the necessary building blocks for creating rewards, but this is *not* designed for production. Users *must* create their own custom reinforcement learning model class or use a pre-built one from outside the Freqtrade source code and save it to `user_data/freqaimodels`. It is inside the `calculate_reward()` where creative theories about the market can be expressed. For example, you can reward your agent when it makes a winning trade, and penalize the agent when it makes a losing trade. Or perhaps, you wish to reward the agent for entering trades, and penalize the agent for sitting in trades too long. Below we show examples of how these rewards are all calculated:
 
 !!! note "Hint"
-    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occuring at a single point in time.
+    The best reward functions are ones that are continuously differentiable, and well scaled. In other words, adding a single large negative penalty to a rare event is not a good idea, and the neural net will not be able to learn that function. Instead, it is better to add a small negative penalty to a common event. This will help the agent learn faster. Not only this, but you can help improve the continuity of your rewards/penalties by having them scale with severity according to some linear/exponential functions. In other words, you'd slowly scale the penalty as the duration of the trade increases. This is better than a single large penalty occurring at a single point in time.
 
 ```python
 from freqtrade.freqai.prediction_models.ReinforcementLearner import ReinforcementLearner
diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md
index e4dc02c76..76023ad4e 100644
--- a/docs/telegram-usage.md
+++ b/docs/telegram-usage.md
@@ -59,7 +59,7 @@ For the Freqtrade configuration, you can then use the the full value (including
    "chat_id": "-1001332619709"
 ```
 !!! Warning "Using telegram groups"
-    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasent surprises.
+    When using telegram groups, you're giving every member of the telegram group access to your freqtrade bot and to all commands possible via telegram. Please make sure that you can trust everyone in the telegram group to avoid unpleasant surprises.
 
 ## Control telegram noise
 
diff --git a/docs/webhook-config.md b/docs/webhook-config.md
index b4044655c..9125ff361 100644
--- a/docs/webhook-config.md
+++ b/docs/webhook-config.md
@@ -65,7 +65,7 @@ You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw
 
 The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel.
 
-When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
+When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be output in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example:
 
 ```json
   "webhook": {

From 865ebc314326cf32af48472d0603416814771f89 Mon Sep 17 00:00:00 2001
From: Alberto <alberto.rinaudo+seiza@gmail.com>
Date: Thu, 7 Mar 2024 17:05:13 +0000
Subject: [PATCH 49/76] update status table to show total amounts in stake
 currency

Signed-off-by: Alberto <alberto.rinaudo+seiza@gmail.com>
---
 freqtrade/rpc/rpc.py  | 6 ++++++
 tests/rpc/test_rpc.py | 8 ++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 6e8447d29..7642d1697 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -291,6 +291,10 @@ class RPC:
                         profit_str += f" ({fiat_profit:.2f})"
                         fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
                             else fiat_profit_sum + fiat_profit
+                else:
+                    profit_str += f" ({trade_profit:.2f})"
+                    fiat_profit_sum = trade_profit if isnan(fiat_profit_sum) \
+                        else fiat_profit_sum + trade_profit
 
                 active_attempt_side_symbols = [
                     '*' if (oo and oo.ft_order_side == trade.entry_side) else '**'
@@ -317,6 +321,8 @@ class RPC:
             profitcol = "Profit"
             if self._fiat_converter:
                 profitcol += " (" + fiat_display_currency + ")"
+            else:
+                profitcol += " (" + stake_currency + ")"
 
             columns = [
                 'ID L/S' if nonspot else 'ID',
diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py
index 85b105892..bc1fc6227 100644
--- a/tests/rpc/test_rpc.py
+++ b/tests/rpc/test_rpc.py
@@ -223,8 +223,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert "Pair" in headers
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
-    assert '0.00' == result[0][3]
-    assert isnan(fiat_profit_sum)
+    assert '0.00 (0.00)' == result[0][3]
+    assert '0.00' == f'{fiat_profit_sum:.2f}'
 
     mocker.patch(f'{EXMS}._dry_is_price_crossed', return_value=True)
     freqtradebot.process()
@@ -234,8 +234,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert "Pair" in headers
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
-    assert '-0.41%' == result[0][3]
-    assert isnan(fiat_profit_sum)
+    assert '-0.41% (-0.00)' == result[0][3]
+    assert '-0.00' == f'{fiat_profit_sum:.2f}'
 
     # Test with fiat convert
     rpc._fiat_converter = CryptoToFiatConverter()

From b690325f2260793b551f9e1c30acf721be2e00be Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 8 Mar 2024 06:39:34 +0100
Subject: [PATCH 50/76] Remove typo in change-dir notebook

closes #9916
---
 docs/strategy_analysis_example.md                   | 2 +-
 freqtrade/templates/strategy_analysis_example.ipynb | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md
index 846c53238..22828b899 100644
--- a/docs/strategy_analysis_example.md
+++ b/docs/strategy_analysis_example.md
@@ -19,7 +19,7 @@ from pathlib import Path
 project_root = "somedir/freqtrade"
 i=0
 try:
-    os.chdirdir(project_root)
+    os.chdir(project_root)
     assert Path('LICENSE').is_file()
 except:
     while i<4 and (not Path('LICENSE').is_file()):
diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb
index 0b30dbd54..8d4459a3c 100644
--- a/freqtrade/templates/strategy_analysis_example.ipynb
+++ b/freqtrade/templates/strategy_analysis_example.ipynb
@@ -35,7 +35,7 @@
     "project_root = \"somedir/freqtrade\"\n",
     "i=0\n",
     "try:\n",
-    "    os.chdirdir(project_root)\n",
+    "    os.chdir(project_root)\n",
     "    assert Path('LICENSE').is_file()\n",
     "except:\n",
     "    while i<4 and (not Path('LICENSE').is_file()):\n",
@@ -181,7 +181,7 @@
     "\n",
     "# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n",
     "backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
-    "# backtest_dir can also point to a specific file \n",
+    "# backtest_dir can also point to a specific file\n",
     "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\""
    ]
   },

From 2cfe9939517d1c7ea1bcee541f458acc70ea349d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 8 Mar 2024 07:10:41 +0100
Subject: [PATCH 51/76] Fix condition for min-stake in position-adjust mode

closes #9915
---
 freqtrade/freqtradebot.py | 2 +-
 freqtrade/rpc/rpc.py      | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
index 974f8124e..8ad151108 100644
--- a/freqtrade/freqtradebot.py
+++ b/freqtrade/freqtradebot.py
@@ -962,7 +962,7 @@ class FreqtradeBot(LoggingMixin):
         # edge-case for now.
         min_stake_amount = self.exchange.get_min_pair_stake_amount(
             pair, enter_limit_requested,
-            self.strategy.stoploss if not mode != 'pos_adjust' else 0.0,
+            self.strategy.stoploss if not mode == 'pos_adjust' else 0.0,
             leverage)
         max_stake_amount = self.exchange.get_max_pair_stake_amount(
             pair, enter_limit_requested, leverage)
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 47646923d..8d91fc92c 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -927,6 +927,7 @@ class RPC:
                                              is_short=is_short,
                                              enter_tag=enter_tag,
                                              leverage_=leverage,
+                                             mode='pos_adjust' if trade else 'initial'
                                              ):
                 Trade.commit()
                 trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()

From acbb485302aeacaf49a07809601d726cd453ab9e Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 15:01:13 +0100
Subject: [PATCH 52/76] Add bot start and bot-startup to health endpoint

---
 freqtrade/rpc/api_server/api_schemas.py |  4 +++
 freqtrade/rpc/rpc.py                    | 43 ++++++++++++++++++-------
 tests/rpc/test_rpc.py                   |  2 ++
 3 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py
index 3ea9ed4d0..af8d8ddf4 100644
--- a/freqtrade/rpc/api_server/api_schemas.py
+++ b/freqtrade/rpc/api_server/api_schemas.py
@@ -559,3 +559,7 @@ class SysInfo(BaseModel):
 class Health(BaseModel):
     last_process: Optional[datetime] = None
     last_process_ts: Optional[int] = None
+    bot_start: Optional[datetime] = None
+    bot_start_ts: Optional[int] = None
+    bot_startup: Optional[datetime] = None
+    bot_startup_ts: Optional[int] = None
diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index 4487814c5..cd30d5be8 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -1366,19 +1366,40 @@ class RPC:
 
     def health(self) -> Dict[str, Optional[Union[str, int]]]:
         last_p = self._freqtrade.last_process
-        if last_p is None:
-            return {
-                "last_process": None,
-                "last_process_loc": None,
-                "last_process_ts": None,
-            }
-
-        return {
-            "last_process": str(last_p),
-            "last_process_loc": format_date(last_p.astimezone(tzlocal())),
-            "last_process_ts": int(last_p.timestamp()),
+        res = {
+            "last_process": None,
+            "last_process_loc": None,
+            "last_process_ts": None,
+            "bot_start": None,
+            "bot_start_loc": None,
+            "bot_start_ts": None,
+            "bot_startup": None,
+            "bot_startup_loc": None,
+            "bot_startup_ts": None,
         }
 
+        if last_p is not None:
+            res.update({
+                "last_process": str(last_p),
+                "last_process_loc": format_date(last_p.astimezone(tzlocal())),
+                "last_process_ts": int(last_p.timestamp()),
+            })
+
+        if (bot_start := KeyValueStore.get_datetime_value(KeyStoreKeys.BOT_START_TIME)):
+            res.update({
+                "bot_start": str(bot_start),
+                "bot_start_loc": format_date(bot_start.astimezone(tzlocal())),
+                "bot_start_ts": int(bot_start.timestamp()),
+            })
+        if (bot_startup := KeyValueStore.get_datetime_value(KeyStoreKeys.STARTUP_TIME)):
+            res.update({
+                "bot_startup": str(bot_startup),
+                "bot_startup_loc": format_date(bot_startup.astimezone(tzlocal())),
+                "bot_startup_ts": int(bot_startup.timestamp()),
+            })
+
+        return res
+
     def _update_market_direction(self, direction: MarketDirection) -> None:
         self._freqtrade.strategy.market_direction = direction
 
diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py
index bc1fc6227..66a750f1f 100644
--- a/tests/rpc/test_rpc.py
+++ b/tests/rpc/test_rpc.py
@@ -10,6 +10,7 @@ from freqtrade.edge import PairInfo
 from freqtrade.enums import SignalDirection, State, TradingMode
 from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError
 from freqtrade.persistence import Order, Trade
+from freqtrade.persistence.key_value_store import set_startup_time
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.rpc import RPC, RPCException
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -1298,6 +1299,7 @@ def test_rpc_health(mocker, default_conf) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
 
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    set_startup_time()
     rpc = RPC(freqtradebot)
     result = rpc.health()
     assert result['last_process'] is None

From f8cbf138ee659238f727fc988b4fbcc414293d82 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 15:47:01 +0100
Subject: [PATCH 53/76] Add Initial bot start and current bot start to /health
 telegram msg

---
 freqtrade/rpc/telegram.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index f7e7362ef..2d59e1f16 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1704,7 +1704,9 @@ class Telegram(RPCHandler):
         Shows the last process timestamp
         """
         health = self._rpc.health()
-        message = f"Last process: `{health['last_process_loc']}`"
+        message = f"Last process: `{health['last_process_loc']}`\n"
+        message += f"Initial bot Start: `{health['bot_start_loc']}`\n"
+        message += f"Current bot Start: `{health['bot_startup_loc']}`"
         await self._send_msg(message)
 
     @authorized_only

From 1b608a162ef8cefd8c8e9803ef8d93bf4ca151ff Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 16:10:33 +0100
Subject: [PATCH 54/76] Add type-hint for result dictionary

---
 freqtrade/rpc/rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py
index cd30d5be8..8bb7f754f 100644
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@@ -1366,7 +1366,7 @@ class RPC:
 
     def health(self) -> Dict[str, Optional[Union[str, int]]]:
         last_p = self._freqtrade.last_process
-        res = {
+        res: Dict[str, Union[None, str, int]] = {
             "last_process": None,
             "last_process_loc": None,
             "last_process_ts": None,

From 29f90cbd048352461728ff04c6870216d6e09078 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 17:34:06 +0100
Subject: [PATCH 55/76] Run CI on macos-14 (M1)

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e0587525e..bb1058afb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -124,7 +124,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ "macos-latest", "macos-13" ]
+        os: [ "macos-latest", "macos-13", "macos-14" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:

From 7cceddb3df169c3bbc598329c94d924d822de3b1 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 17:35:14 +0100
Subject: [PATCH 56/76] Improve wording on /health message

---
 freqtrade/rpc/telegram.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py
index 2d59e1f16..f99149c01 100644
--- a/freqtrade/rpc/telegram.py
+++ b/freqtrade/rpc/telegram.py
@@ -1705,8 +1705,8 @@ class Telegram(RPCHandler):
         """
         health = self._rpc.health()
         message = f"Last process: `{health['last_process_loc']}`\n"
-        message += f"Initial bot Start: `{health['bot_start_loc']}`\n"
-        message += f"Current bot Start: `{health['bot_startup_loc']}`"
+        message += f"Initial bot start: `{health['bot_start_loc']}`\n"
+        message += f"Last bot restart: `{health['bot_startup_loc']}`"
         await self._send_msg(message)
 
     @authorized_only

From 86db8883862aa7e6d44ec9d994ffcc1cd3f234c6 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 17:41:10 +0100
Subject: [PATCH 57/76] Install libomp from brew for macos

closes #9874
---
 setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.sh b/setup.sh
index 6bf85edab..68374a689 100755
--- a/setup.sh
+++ b/setup.sh
@@ -161,7 +161,7 @@ function install_macos() {
         /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     fi
 
-    brew install gettext
+    brew install gettext libomp
 
     #Gets number after decimal in python version
     version=$(egrep -o 3.\[0-9\]+ <<< $PYTHON | sed 's/3.//g')

From cc3b2000eb1f13dd22e1cfbb370d3993a65319ca Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 17:55:22 +0100
Subject: [PATCH 58/76] Avoid fully patching torch on M1 macs

---
 tests/freqai/conftest.py              |  7 ++++++-
 tests/freqai/test_freqai_interface.py | 10 ++--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py
index 81d72d92a..55f0296a3 100644
--- a/tests/freqai/conftest.py
+++ b/tests/freqai/conftest.py
@@ -25,10 +25,15 @@ def is_mac() -> bool:
     return "Darwin" in machine
 
 
+def is_arm() -> bool:
+    machine = platform.machine()
+    return "arm" in machine or "aarch64" in machine
+
+
 @pytest.fixture(autouse=True)
 def patch_torch_initlogs(mocker) -> None:
 
-    if is_mac():
+    if is_mac() and not is_arm():
         # Mock torch import completely
         import sys
         import types
diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py
index 58648d97f..2a71e8af6 100644
--- a/tests/freqai/test_freqai_interface.py
+++ b/tests/freqai/test_freqai_interface.py
@@ -1,5 +1,4 @@
 import logging
-import platform
 import shutil
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -15,13 +14,8 @@ from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re
-from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, is_py12, make_rl_config,
-                                   mock_pytorch_mlp_model_training_parameters)
-
-
-def is_arm() -> bool:
-    machine = platform.machine()
-    return "arm" in machine or "aarch64" in machine
+from tests.freqai.conftest import (get_patched_freqai_strategy, is_arm, is_mac, is_py12,
+                                   make_rl_config, mock_pytorch_mlp_model_training_parameters)
 
 
 def can_run_model(model: str) -> None:

From 971a81e15d5191a9b72a7e681bde3258688f7ffe Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 18:15:47 +0100
Subject: [PATCH 59/76] Bump catboost to 1.2.3, remove 3.12 restriction

---
 requirements-freqai.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-freqai.txt b/requirements-freqai.txt
index 0532562da..31366efa7 100644
--- a/requirements-freqai.txt
+++ b/requirements-freqai.txt
@@ -5,7 +5,7 @@
 # Required for freqai
 scikit-learn==1.4.1.post1
 joblib==1.3.2
-catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12'
+catboost==1.2.3; 'arm' not in platform_machine
 lightgbm==4.3.0
 xgboost==2.0.3
 tensorboard==2.16.2

From c5f2a69d9c9836b2cf905a70fbccd5dc08c69eee Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 18:16:22 +0100
Subject: [PATCH 60/76] Allow running catboost tests on 3.12

---
 tests/freqai/test_freqai_interface.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py
index 58648d97f..cceda8e8e 100644
--- a/tests/freqai/test_freqai_interface.py
+++ b/tests/freqai/test_freqai_interface.py
@@ -27,9 +27,6 @@ def is_arm() -> bool:
 def can_run_model(model: str) -> None:
     is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model
 
-    if is_py12() and ("Catboost" in model or is_pytorch_model):
-        pytest.skip("Model not supported on python 3.12 yet.")
-
     if is_arm() and "Catboost" in model:
         pytest.skip("CatBoost is not supported on ARM.")
 

From edc74ae2e460bb2c6bf051c11cb4a58d122d90d2 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 18:23:35 +0100
Subject: [PATCH 61/76] Split macos Installation into 2 separate actions

---
 .github/workflows/ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bb1058afb..fc3886b32 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -154,7 +154,7 @@ jobs:
       run: |
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
-    - name: Installation - macOS
+    - name: Installation - macOS (Brew)
       run: |
         # brew update
         # TODO: Should be the brew upgrade
@@ -177,6 +177,9 @@ jobs:
         rm /usr/local/bin/python3.12-config || true
 
         brew install hdf5 c-blosc libomp
+
+    - name: Installation (python)
+      run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib

From cb1f49e81ce142f830f5b0dd0f359a0171fea4b0 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 19:25:17 +0100
Subject: [PATCH 62/76] Don't run torch models on 3.12 yet

---
 tests/freqai/test_freqai_interface.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py
index cceda8e8e..e3c286fcd 100644
--- a/tests/freqai/test_freqai_interface.py
+++ b/tests/freqai/test_freqai_interface.py
@@ -27,6 +27,9 @@ def is_arm() -> bool:
 def can_run_model(model: str) -> None:
     is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model
 
+    if is_py12() and is_pytorch_model:
+        pytest.skip("Model not supported on python 3.12 yet.")
+
     if is_arm() and "Catboost" in model:
         pytest.skip("CatBoost is not supported on ARM.")
 

From 0bd50a6e2479e68caeed95f269f3a80447b65f75 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 19:44:40 +0100
Subject: [PATCH 63/76] Don't disable tensorboard on mac ARM

---
 tests/freqai/test_freqai_interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py
index 2a71e8af6..fb7af5853 100644
--- a/tests/freqai/test_freqai_interface.py
+++ b/tests/freqai/test_freqai_interface.py
@@ -237,7 +237,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
 def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog):
     can_run_model(model)
     test_tb = True
-    if is_mac():
+    if is_mac() and not is_arm():
         test_tb = False
 
     freqai_conf.get("freqai", {}).update({"save_backtest_models": True})

From 4e94178169f910c7a665f689d3a625f7d7b43449 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sat, 9 Mar 2024 20:10:40 +0100
Subject: [PATCH 64/76] exclude python 3.9 on Macos 14

---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fc3886b32..44f489346 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -126,6 +126,9 @@ jobs:
       matrix:
         os: [ "macos-latest", "macos-13", "macos-14" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
+        exclude:
+          - os: "macos-14"
+            python-version: "3.9"
 
     steps:
     - uses: actions/checkout@v4

From 518b6eb56577385c3b358e35c9e714a4f7d18451 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Sun, 10 Mar 2024 19:31:43 +0100
Subject: [PATCH 65/76] use dt_ts to simplify exchange date math

---
 freqtrade/exchange/exchange.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py
index d17b442ab..482ac598f 100644
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@@ -659,7 +659,7 @@ class Exchange:
 
         candle_limit = self.ohlcv_candle_limit(
             timeframe, self._config['candle_type_def'],
-            int(date_minus_candles(timeframe, startup_candles).timestamp() * 1000)
+            dt_ts(date_minus_candles(timeframe, startup_candles))
             if timeframe else None)
         # Require one more candle - to account for the still open candle.
         candle_count = startup_candles + 1
@@ -2043,7 +2043,7 @@ class Exchange:
                 timeframe, candle_type, since_ms)
             move_to = one_call * self.required_candle_call_count
             now = timeframe_to_next_date(timeframe)
-            since_ms = int((now - timedelta(seconds=move_to // 1000)).timestamp() * 1000)
+            since_ms = dt_ts(now - timedelta(seconds=move_to // 1000))
 
         if since_ms:
             return self._async_get_historic_ohlcv(
@@ -2503,7 +2503,7 @@ class Exchange:
             )
 
         if type(since) is datetime:
-            since = int(since.timestamp()) * 1000   # * 1000 for ms
+            since = dt_ts(since)
 
         try:
             funding_history = self._api.fetch_funding_history(
@@ -2833,7 +2833,7 @@ class Exchange:
 
         if not close_date:
             close_date = datetime.now(timezone.utc)
-        since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
+        since_ms = dt_ts(timeframe_to_prev_date(timeframe, open_date))
 
         mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type)
         funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE)

From 60b9d9448ac925b4ca7b38d58e8cbe8d094ccab8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:37:19 +0000
Subject: [PATCH 66/76] Bump the types group with 2 updates

Bumps the types group with 2 updates: [types-requests](https://github.com/python/typeshed) and [types-python-dateutil](https://github.com/python/typeshed).


Updates `types-requests` from 2.31.0.20240218 to 2.31.0.20240311
- [Commits](https://github.com/python/typeshed/commits)

Updates `types-python-dateutil` from 2.8.19.20240106 to 2.8.19.20240311
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-requests
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: types
- dependency-name: types-python-dateutil
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: types
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements-dev.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 562841375..b7d2ae079 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -26,6 +26,6 @@ nbconvert==7.16.1
 # mypy types
 types-cachetools==5.3.0.7
 types-filelock==3.2.7
-types-requests==2.31.0.20240218
+types-requests==2.31.0.20240311
 types-tabulate==0.9.0.20240106
-types-python-dateutil==2.8.19.20240106
+types-python-dateutil==2.8.19.20240311

From 80560a389c0a3cfd2f58226085c1453a1eac3634 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:39:48 +0000
Subject: [PATCH 67/76] Bump mypy from 1.8.0 to 1.9.0

Bumps [mypy](https://github.com/python/mypy) from 1.8.0 to 1.9.0.
- [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md)
- [Commits](https://github.com/python/mypy/compare/v1.8.0...1.9.0)

---
updated-dependencies:
- dependency-name: mypy
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 562841375..455ce8ccd 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -8,7 +8,7 @@
 
 coveralls==3.3.1
 ruff==0.3.0
-mypy==1.8.0
+mypy==1.9.0
 pre-commit==3.6.2
 pytest==8.1.0
 pytest-asyncio==0.23.5

From 6d2f454d8cccefadad204e42a35219f0f4d28cfc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:40:01 +0000
Subject: [PATCH 68/76] Bump pymdown-extensions from 10.7 to 10.7.1

Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 10.7 to 10.7.1.
- [Release notes](https://github.com/facelessuser/pymdown-extensions/releases)
- [Commits](https://github.com/facelessuser/pymdown-extensions/compare/10.7...10.7.1)

---
updated-dependencies:
- dependency-name: pymdown-extensions
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 docs/requirements-docs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
index 55a2c11aa..be276085e 100644
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@@ -2,5 +2,5 @@ markdown==3.5.2
 mkdocs==1.5.3
 mkdocs-material==9.5.12
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.7
+pymdown-extensions==10.7.1
 jinja2==3.1.3

From e1fdb8dec9bb5e6584bd0010880dbefa97739632 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:40:06 +0000
Subject: [PATCH 69/76] Bump nbconvert from 7.16.1 to 7.16.2

Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.16.1 to 7.16.2.
- [Release notes](https://github.com/jupyter/nbconvert/releases)
- [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md)
- [Commits](https://github.com/jupyter/nbconvert/compare/v7.16.1...v7.16.2)

---
updated-dependencies:
- dependency-name: nbconvert
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements-dev.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 562841375..3e56e6633 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -21,7 +21,7 @@ isort==5.13.2
 time-machine==2.14.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.16.1
+nbconvert==7.16.2
 
 # mypy types
 types-cachetools==5.3.0.7

From ab6a5d75bcab19b6d87c7d40cb22bb5ce2aafed6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:40:15 +0000
Subject: [PATCH 70/76] Bump python-telegram-bot from 20.8 to 21.0.1

Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 20.8 to 21.0.1.
- [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases)
- [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst)
- [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v20.8...v21.0.1)

---
updated-dependencies:
- dependency-name: python-telegram-bot
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 44e8e2ccb..1a7ce1f71 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ ccxt==4.2.58
 cryptography==42.0.5
 aiohttp==3.9.3
 SQLAlchemy==2.0.27
-python-telegram-bot==20.8
+python-telegram-bot==21.0.1
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
 arrow==1.3.0

From 9b44d1d8cb0a8dcdd9951428073221bc0012b070 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:40:29 +0000
Subject: [PATCH 71/76] Bump packaging from 23.2 to 24.0

Bumps [packaging](https://github.com/pypa/packaging) from 23.2 to 24.0.
- [Release notes](https://github.com/pypa/packaging/releases)
- [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pypa/packaging/compare/23.2...24.0)

---
updated-dependencies:
- dependency-name: packaging
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 44e8e2ccb..c39d5b254 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -60,4 +60,4 @@ websockets==12.0
 janus==1.0.0
 
 ast-comments==1.2.1
-packaging==23.2
+packaging==24.0

From 23d226d372e5d3807675b1343ea19c35323655b5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 03:46:55 +0000
Subject: [PATCH 72/76] Bump pypa/gh-action-pypi-publish from 1.8.12 to 1.8.14

Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.12 to 1.8.14.
- [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases)
- [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.12...v1.8.14)

---
updated-dependencies:
- dependency-name: pypa/gh-action-pypi-publish
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 44f489346..a268acf80 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -488,12 +488,12 @@ jobs:
         path: dist
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.12
+      uses: pypa/gh-action-pypi-publish@v1.8.14
       with:
         repository-url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.12
+      uses: pypa/gh-action-pypi-publish@v1.8.14
 
 
   deploy-docker:

From c78480c4941878c5aaa00a848d557a08a432665f Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Mon, 11 Mar 2024 06:28:11 +0100
Subject: [PATCH 73/76] Bump types in pre-commit file

---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 23fa54326..facc774f3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -16,9 +16,9 @@ repos:
         additional_dependencies:
           - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.20240218
+          - types-requests==2.31.0.20240311
           - types-tabulate==0.9.0.20240106
-          - types-python-dateutil==2.8.19.20240106
+          - types-python-dateutil==2.8.19.20240311
           - SQLAlchemy==2.0.27
         # stages: [push]
 

From 1c91675c5840a442bad137d3d49548d343c1e075 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 06:14:04 +0000
Subject: [PATCH 74/76] Bump the pytest group with 2 updates

Bumps the pytest group with 2 updates: [pytest](https://github.com/pytest-dev/pytest) and [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio).


Updates `pytest` from 8.1.0 to 8.1.1
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/8.1.0...8.1.1)

Updates `pytest-asyncio` from 0.23.5 to 0.23.5.post1
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.23.5...v0.23.5.post1)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: pytest
- dependency-name: pytest-asyncio
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: pytest
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements-dev.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 455ce8ccd..dd41a80db 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -10,8 +10,8 @@ coveralls==3.3.1
 ruff==0.3.0
 mypy==1.9.0
 pre-commit==3.6.2
-pytest==8.1.0
-pytest-asyncio==0.23.5
+pytest==8.1.1
+pytest-asyncio==0.23.5.post1
 pytest-cov==4.1.0
 pytest-mock==3.12.0
 pytest-random-order==1.1.1

From 33556f3c2c5343e7bb06fe148b0605fd4bb6ed30 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 07:11:44 +0000
Subject: [PATCH 75/76] Bump mkdocs-material from 9.5.12 to 9.5.13

Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.12 to 9.5.13.
- [Release notes](https://github.com/squidfunk/mkdocs-material/releases)
- [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG)
- [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.12...9.5.13)

---
updated-dependencies:
- dependency-name: mkdocs-material
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 docs/requirements-docs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
index be276085e..33d58fdbb 100644
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@@ -1,6 +1,6 @@
 markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.12
+mkdocs-material==9.5.13
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.7.1
 jinja2==3.1.3

From 018d10b3461acb2708d4ff84380d42133d390142 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 07:33:37 +0000
Subject: [PATCH 76/76] Bump ccxt from 4.2.58 to 4.2.66

Bumps [ccxt](https://github.com/ccxt/ccxt) from 4.2.58 to 4.2.66.
- [Release notes](https://github.com/ccxt/ccxt/releases)
- [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ccxt/ccxt/compare/4.2.58...4.2.66)

---
updated-dependencies:
- dependency-name: ccxt
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 1a7ce1f71..d9528f7ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ numpy==1.26.4
 pandas==2.2.1
 pandas-ta==0.3.14b
 
-ccxt==4.2.58
+ccxt==4.2.66
 cryptography==42.0.5
 aiohttp==3.9.3
 SQLAlchemy==2.0.27