Merge pull request #11093 from arenstar/api-server-list-custom-data

feat: api_server and client supporting list_custom_data

docs/rest-api.md

@@ -302,6 +302,19 @@ trades
         :param limit: Limits trades to the X last trades. Max 500 trades.
         :param offset: Offset by this amount of trades.
 
+list_open_trades_custom_data
+    Return a dict containing open trades custom-datas
+
+        :param key: str, optional - Key of the custom-data
+        :param limit: Limits trades to X trades.
+        :param offset: Offset by this amount of trades.
+
+list_custom_data
+    Return a dict containing custom-datas of a specified trade
+
+        :param trade_id: int - ID of the trade
+        :param key: str, optional - Key of the custom-data
+
 version
 	Return the version of the bot.
 

freqtrade/persistence/trade_model.py

@@ -1353,8 +1353,10 @@ class LocalTrade:
 
     def get_custom_data(self, key: str, default: Any = None) -> Any:
         """
-        Get custom data for this trade
+        Get custom data for this trade.
+
         :param key: key of the custom data
+        :param default: value to return if no data is found
         """
         data = CustomDataWrapper.get_custom_data(trade_id=self.id, key=key)
         if data:

freqtrade/rpc/api_server/api_schemas.py

@@ -637,3 +637,16 @@ class Health(BaseModel):
     bot_start_ts: int | None = None
     bot_startup: datetime | None = None
     bot_startup_ts: int | None = None
+
+
+class CustomDataEntry(BaseModel):
+    key: str
+    type: str
+    value: Any
+    created_at: datetime
+    updated_at: datetime | None = None
+
+
+class ListCustomData(BaseModel):
+    trade_id: int
+    custom_data: list[CustomDataEntry]

freqtrade/rpc/api_server/api_v1.py

@@ -29,6 +29,7 @@ from freqtrade.rpc.api_server.api_schemas import (
     FreqAIModelListResponse,
     Health,
     HyperoptLossListResponse,
+    ListCustomData,
     Locks,
     LocksPayload,
     Logs,
@@ -213,6 +214,36 @@ def trade_reload(tradeid: int, rpc: RPC = Depends(get_rpc)):
     return rpc._rpc_trade_status([tradeid])[0]
 
 
+@router.get("/trades/open/custom-data", response_model=list[ListCustomData], tags=["trading"])
+def list_open_trades_custom_data(
+    key: str | None = Query(None, description="Optional key to filter data"),
+    limit: int = Query(100, ge=1, description="Maximum number of different trades to return data"),
+    offset: int = Query(0, ge=0, description="Number of trades to skip for pagination"),
+    rpc: RPC = Depends(get_rpc),
+):
+    """
+    Fetch custom data for all open trades.
+    If a key is provided, it will be used to filter data accordingly.
+    Pagination is implemented via the `limit` and `offset` parameters.
+    """
+    try:
+        return rpc._rpc_list_custom_data(key=key, limit=limit, offset=offset)
+    except RPCException as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@router.get("/trades/{trade_id}/custom-data", response_model=list[ListCustomData], tags=["trading"])
+def list_custom_data(trade_id: int, key: str | None = Query(None), rpc: RPC = Depends(get_rpc)):
+    """
+    Fetch custom data for a specific trade.
+    If a key is provided, it will be used to filter data accordingly.
+    """
+    try:
+        return rpc._rpc_list_custom_data(trade_id, key=key)
+    except RPCException as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
 # TODO: Missing response model
 @router.get("/edge", tags=["info"])
 def edge(rpc: RPC = Depends(get_rpc)):

freqtrade/rpc/rpc.py

@@ -33,7 +33,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.exchange.exchange_utils import price_to_precision
 from freqtrade.loggers import bufferHandler
-from freqtrade.persistence import KeyStoreKeys, KeyValueStore, PairLocks, Trade
+from freqtrade.persistence import CustomDataWrapper, KeyStoreKeys, KeyValueStore, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -1115,31 +1115,70 @@ class RPC:
                 "cancel_order_count": c_count,
             }
 
-    def _rpc_list_custom_data(self, trade_id: int, key: str | None) -> list[dict[str, Any]]:
-        # Query for trade
-        trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
-        if trade is None:
-            return []
-        # Query custom_data
-        custom_data = []
-        if key:
-            data = trade.get_custom_data(key=key)
-            if data:
-                custom_data = [data]
+    def _rpc_list_custom_data(
+        self, trade_id: int | None = None, key: str | None = None, limit: int = 100, offset: int = 0
+    ) -> list[dict[str, Any]]:
+        """
+        Fetch custom data for a specific trade, or all open trades if `trade_id` is not provided.
+        Pagination is applied via `limit` and `offset`.
+
+        Returns an array of dictionaries, each containing:
+        - "trade_id": the ID of the trade (int)
+        - "custom_data": a list of custom data dicts, each with the fields:
+                "id", "key", "type", "value", "created_at", "updated_at"
+        """
+        trades: Sequence[Trade]
+        if trade_id is None:
+            # Get all open trades
+            trades = Trade.session.scalars(
+                Trade.get_trades_query([Trade.is_open.is_(True)])
+                .order_by(Trade.id)
+                .limit(limit)
+                .offset(offset)
+            ).all()
         else:
-            custom_data = trade.get_all_custom_data()
-        return [
-            {
-                "id": data_entry.id,
-                "ft_trade_id": data_entry.ft_trade_id,
-                "cd_key": data_entry.cd_key,
-                "cd_type": data_entry.cd_type,
-                "cd_value": data_entry.cd_value,
-                "created_at": data_entry.created_at,
-                "updated_at": data_entry.updated_at,
-            }
-            for data_entry in custom_data
-        ]
+            trades = Trade.get_trades(trade_filter=[Trade.id == trade_id]).all()
+
+        if not trades:
+            raise RPCException(
+                f"No trade found for trade_id: {trade_id}" if trade_id else "No open trades found."
+            )
+
+        results = []
+        for trade in trades:
+            # Depending on whether a specific key is provided, retrieve custom data accordingly.
+            if key:
+                data = trade.get_custom_data_entry(key=key)
+                # If data exists, wrap it in a list so the output remains consistent.
+                custom_data = [data] if data else []
+            else:
+                custom_data = trade.get_all_custom_data()
+
+            # Format and Append result for the trade if any custom data was found.
+            if custom_data:
+                formatted_custom_data = [
+                    {
+                        "key": data_entry.cd_key,
+                        "type": data_entry.cd_type,
+                        "value": CustomDataWrapper._convert_custom_data(data_entry).value,
+                        "created_at": data_entry.created_at,
+                        "updated_at": data_entry.updated_at,
+                    }
+                    for data_entry in custom_data
+                ]
+                results.append({"trade_id": trade.id, "custom_data": formatted_custom_data})
+
+            # Handle case when there is no custom data found across trades.
+            if not results:
+                message_details = ""
+                if key:
+                    message_details += f"with key '{key}' "
+                message_details += (
+                    f"found for Trade ID: {trade_id}." if trade_id else "found for any open trades."
+                )
+                raise RPCException(f"No custom-data {message_details}")
+
+        return results
 
     def _rpc_performance(self) -> list[dict[str, Any]]:
         """

freqtrade/rpc/telegram.py

@@ -1981,16 +1981,17 @@ class Telegram(RPCHandler):
             results = self._rpc._rpc_list_custom_data(trade_id, key)
             messages = []
             if len(results) > 0:
-                messages.append("Found custom-data entr" + ("ies: " if len(results) > 1 else "y: "))
-                for result in results:
+                trade_custom_data = results[0]["custom_data"]
+                messages.append(
+                    "Found custom-data entr" + ("ies: " if len(trade_custom_data) > 1 else "y: ")
+                )
+                for custom_data in trade_custom_data:
                     lines = [
-                        f"*Key:* `{result['cd_key']}`",
-                        f"*ID:* `{result['id']}`",
-                        f"*Trade ID:* `{result['ft_trade_id']}`",
-                        f"*Type:* `{result['cd_type']}`",
-                        f"*Value:* `{result['cd_value']}`",
-                        f"*Create Date:* `{format_date(result['created_at'])}`",
-                        f"*Update Date:* `{format_date(result['updated_at'])}`",
+                        f"*Key:* `{custom_data['key']}`",
+                        f"*Type:* `{custom_data['type']}`",
+                        f"*Value:* `{custom_data['value']}`",
+                        f"*Create Date:* `{format_date(custom_data['created_at'])}`",
+                        f"*Update Date:* `{format_date(custom_data['updated_at'])}`",
                     ]
                     # Filter empty lines using list-comprehension
                     messages.append("\n".join([line for line in lines if line]))

ft_client/freqtrade_client/ft_rest_client.py

@@ -269,6 +269,36 @@ class FtRestClient:
             params["offset"] = offset
         return self._get("trades", params)
 
+    def list_open_trades_custom_data(self, key=None, limit=100, offset=0):
+        """List open trades custom-data of the running bot.
+
+        :param key: str, optional - Key of the custom-data
+        :param limit: limit of trades
+        :param offset: trades offset for pagination
+        :return: json object
+        """
+        params = {}
+        params["limit"] = limit
+        params["offset"] = offset
+        if key is not None:
+            params["key"] = key
+
+        return self._get("trades/open/custom-data", params=params)
+
+    def list_custom_data(self, trade_id, key=None):
+        """List custom-data of the running bot for a specific trade.
+
+        :param trade_id: ID of the trade
+        :param key: str, optional - Key of the custom-data
+        :return: JSON object
+        """
+        params = {}
+        params["trade_id"] = trade_id
+        if key is not None:
+            params["key"] = key
+
+        return self._get(f"trades/{trade_id}/custom-data", params=params)
+
     def trade(self, trade_id):
         """Return specific trade
 

tests/rpc/test_rpc_apiserver.py

@@ -24,7 +24,7 @@ from freqtrade.enums import CandleType, RunMode, State, TradingMode
 from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException
 from freqtrade.loggers import setup_logging, setup_logging_pre
 from freqtrade.optimize.backtesting import Backtesting
-from freqtrade.persistence import Trade
+from freqtrade.persistence import CustomDataWrapper, Trade
 from freqtrade.rpc import RPC
 from freqtrade.rpc.api_server import ApiServer
 from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token
@@ -802,6 +802,211 @@ def test_api_trade_single(botclient, mocker, fee, ticker, markets, is_short):
     assert rc.json()["is_short"] == is_short
 
 
+@pytest.mark.usefixtures("init_persistence")
+def test_api_custom_data_single_trade(botclient, fee):
+    Trade.reset_trades()
+    CustomDataWrapper.reset_custom_data()
+
+    create_mock_trades_usdt(fee, use_db=True)
+
+    trade1 = Trade.get_trades_proxy()[0]
+
+    assert trade1.get_all_custom_data() == []
+
+    trade1.set_custom_data("test_str", "test_value")
+    trade1.set_custom_data("test_int", 0)
+    trade1.set_custom_data("test_float", 1.54)
+    trade1.set_custom_data("test_bool", True)
+    trade1.set_custom_data("test_dict", {"test": "vl"})
+
+    trade1.set_custom_data("test_int", 1)
+
+    _, client = botclient
+
+    # CASE 1 Checking all custom data of trade 1
+    rc = client_get(client, f"{BASE_URI}/trades/1/custom-data")
+    assert_response(rc)
+
+    # Validate response JSON structure
+    response_json = rc.json()
+
+    assert len(response_json) == 1
+
+    res_cust_data = response_json[0]["custom_data"]
+    expected_data_td_1 = [
+        {"key": "test_str", "type": "str", "value": "test_value"},
+        {"key": "test_int", "type": "int", "value": 1},
+        {"key": "test_float", "type": "float", "value": 1.54},
+        {"key": "test_bool", "type": "bool", "value": True},
+        {"key": "test_dict", "type": "dict", "value": {"test": "vl"}},
+    ]
+
+    # Ensure response contains exactly the expected number of entries
+    assert len(res_cust_data) == len(expected_data_td_1), (
+        f"Expected {len(expected_data_td_1)} entries, but got {len(res_cust_data)}.\n"
+    )
+
+    # Validate each expected entry
+    for expected in expected_data_td_1:
+        matched_item = None
+        for item in res_cust_data:
+            if item["key"] == expected["key"]:
+                matched_item = item
+                break
+
+        assert matched_item is not None, (
+            f"Missing expected entry for key '{expected['key']}'\nExpected: {expected}\n"
+        )
+
+        # Validate individual fields and print only incorrect values
+        mismatches = []
+        for field in ["key", "type", "value"]:
+            if matched_item[field] != expected[field]:
+                mismatches.append(f"{field}: Expected {expected[field]}, Got {matched_item[field]}")
+
+        assert not mismatches, f"Error in entry '{expected['key']}':\n" + "\n".join(mismatches)
+
+    # CASE 2 Checking specific existing key custom data of trade 1
+    rc = client_get(client, f"{BASE_URI}/trades/1/custom-data?key=test_dict")
+    assert_response(rc, 200)
+
+    # CASE 3 Checking specific not existing key custom data of trade 1
+    rc = client_get(client, f"{BASE_URI}/trades/1/custom-data&key=test")
+    assert_response(rc, 404)
+
+    # CASE 4 Trying to get custom-data from not existing trade
+    rc = client_get(client, f"{BASE_URI}/trades/13/custom-data")
+    assert_response(rc, 404)
+    assert rc.json()["detail"] == "No trade found for trade_id: 13"
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_api_custom_data_multiple_open_trades(botclient, fee):
+    use_db = True
+    Trade.use_db = use_db
+    Trade.reset_trades()
+    CustomDataWrapper.reset_custom_data()
+    create_mock_trades(fee, False, use_db)
+    trades = Trade.get_trades_proxy()
+    assert len(trades) == 6
+
+    assert isinstance(trades[0], Trade)
+
+    trades = Trade.get_trades_proxy(is_open=True)
+    assert len(trades) == 4
+
+    create_mock_trades_usdt(fee, use_db=True)
+
+    trade1 = Trade.get_trades_proxy(is_open=True)[0]
+    trade2 = Trade.get_trades_proxy(is_open=True)[1]
+
+    # Initially, no custom data should be present.
+    assert trade1.get_all_custom_data() == []
+    assert trade2.get_all_custom_data() == []
+
+    # Set custom data for the two open trades.
+    trade1.set_custom_data("test_str", "test_value_t1")
+    trade1.set_custom_data("test_float", 1.54)
+    trade1.set_custom_data("test_dict", {"test_t1": "vl_t1"})
+
+    trade2.set_custom_data("test_str", "test_value_t2")
+    trade2.set_custom_data("test_float", 1.55)
+    trade2.set_custom_data("test_dict", {"test_t2": "vl_t2"})
+
+    _, client = botclient
+
+    # CASE 1: Checking all custom data for both trades.
+    rc = client_get(client, f"{BASE_URI}/trades/open/custom-data")
+    assert_response(rc)
+
+    response_json = rc.json()
+
+    # Expecting two trade entries in the response
+    assert len(response_json) == 2, f"Expected 2 trade entries, but got {len(response_json)}.\n"
+
+    # Define expected custom data for each trade.
+    # The keys now use the actual trade_ids from the custom data.
+    expected_custom_data = {
+        1: [
+            {
+                "key": "test_str",
+                "type": "str",
+                "value": "test_value_t1",
+            },
+            {
+                "key": "test_float",
+                "type": "float",
+                "value": 1.54,
+            },
+            {
+                "key": "test_dict",
+                "type": "dict",
+                "value": {"test_t1": "vl_t1"},
+            },
+        ],
+        4: [
+            {
+                "key": "test_str",
+                "type": "str",
+                "value": "test_value_t2",
+            },
+            {
+                "key": "test_float",
+                "type": "float",
+                "value": 1.55,
+            },
+            {
+                "key": "test_dict",
+                "type": "dict",
+                "value": {"test_t2": "vl_t2"},
+            },
+        ],
+    }
+
+    # Iterate over each trade's data in the response and validate entries.
+    for trade_entry in response_json:
+        trade_id = trade_entry.get("trade_id")
+        assert trade_id in expected_custom_data, f"\nUnexpected trade_id: {trade_id}"
+
+        custom_data_list = trade_entry.get("custom_data")
+        expected_data = expected_custom_data[trade_id]
+        assert len(custom_data_list) == len(expected_data), (
+            f"Error for trade_id {trade_id}: "
+            f"Expected {len(expected_data)} entries, but got {len(custom_data_list)}.\n"
+        )
+
+        # For each expected entry, check that the response contains the correct entry.
+        for expected in expected_data:
+            matched_item = None
+            for item in custom_data_list:
+                if item["key"] == expected["key"]:
+                    matched_item = item
+                    break
+
+            assert matched_item is not None, (
+                f"For trade_id {trade_id}, "
+                f"missing expected entry for key '{expected['key']}'\n"
+                f"Expected: {expected}\n"
+            )
+
+            # Validate key fields.
+            mismatches = []
+            for field in ["key", "type", "value"]:
+                if matched_item[field] != expected[field]:
+                    mismatches.append(
+                        f"{field}: Expected {expected[field]}, Got {matched_item[field]}"
+                    )
+            # Check for field presence of created_at and updated_at without comparing values.
+            for field in ["created_at", "updated_at"]:
+                if field not in matched_item:
+                    mismatches.append(f"Missing field: {field}")
+
+            assert not mismatches, (
+                f"Error in entry '{expected['key']}' for trade_id {trade_id}:\n"
+                + "\n".join(mismatches)
+            )
+
+
 @pytest.mark.parametrize("is_short", [True, False])
 def test_api_delete_trade(botclient, mocker, fee, markets, is_short):
     ftbot, client = botclient

tests/rpc/test_rpc_telegram.py

@@ -2903,9 +2903,7 @@ async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee,
     context.args = ["1"]
     await telegram._list_custom_data(update=update, context=context)
     assert msg_mock.call_count == 1
-    assert (
-        "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0]
-    )
+    assert "No custom-data found for Trade ID: 1." in msg_mock.call_args_list[0][0][0]
     msg_mock.reset_mock()
 
     # Add some custom data
@@ -2918,11 +2916,10 @@ async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee,
     assert msg_mock.call_count == 3
     assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0]
     assert (
-        "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n*Value:* `1`\n*Create Date:*"
+        "*Key:* `test_int`\n*Type:* `int`\n*Value:* `1`\n*Create Date:*"
     ) in msg_mock.call_args_list[1][0][0]
     assert (
-        "*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n"
-        '*Value:* `{"test": "dict"}`\n*Create Date:* `'
+        "*Key:* `test_dict`\n*Type:* `dict`\n*Value:* `{'test': 'dict'}`\n*Create Date:* `"
     ) in msg_mock.call_args_list[2][0][0]
 
     msg_mock.reset_mock()