add test_refresh_latest_trades to test_exchange

tests/conftest.py

@@ -574,7 +574,8 @@ def get_default_conf(testdatadir):
             "pair_blacklist": [
                 "DOGE/BTC",
                 "HOT/BTC",
-            ]
+            ],
+            "use_public_trades": True,
         },
         "pairlists": [
             {"method": "StaticPairList"}
@@ -596,6 +597,7 @@ def get_default_conf(testdatadir):
         "internals": {},
         "export": "none",
         "dataformat_ohlcv": "feather",
+        "dataformat_trades": "feather",
         "runmode": "dry_run",
         "candle_type_def": CandleType.SPOT,
     }

tests/exchange/test_exchange.py

@@ -8,8 +8,9 @@ from unittest.mock import MagicMock, Mock, PropertyMock, patch
 import ccxt
 import pytest
 from numpy import NaN
-from pandas import DataFrame
+from pandas import DataFrame, to_datetime
 
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
 from freqtrade.enums import CandleType, MarginMode, RunMode, TradingMode
 from freqtrade.exceptions import (ConfigurationError, DDosProtection, DependencyException,
                                   ExchangeError, InsufficientFundsError, InvalidOrderException,
@@ -2203,15 +2204,173 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None
     caplog.clear()
 
     # Call with invalid timeframe
-    res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '3m', candle_type)], cache=False)
+    res = exchange.refresh_latest_ohlcv([("IOTA/ETH", "3m", candle_type)], cache=False)
     if candle_type != CandleType.MARK:
         assert not res
         assert len(res) == 0
-        assert log_has_re(r'Cannot download \(IOTA\/ETH, 3m\).*', caplog)
+        assert log_has_re(r"Cannot download \(IOTA\/ETH, 3m\).*", caplog)
     else:
         assert len(res) == 1
 
 
+@pytest.mark.parametrize("candle_type", [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT])
+def test_refresh_latest_trades(mocker, default_conf, caplog, candle_type) -> None:
+    # TODO: mock cached trades
+    trades = [
+        {
+            # unix timestamp ms
+            "timestamp": dt_ts(dt_now() - timedelta(minutes=5)),
+            "amount": 16.512,
+            "cost": 10134.07488,
+            "fee": None,
+            "fees": [],
+            "id": "354669639",
+            "order": None,
+            "price": 613.74,
+            "side": "sell",
+            "takerOrMaker": None,
+            "type": None,
+        },
+        {
+            "timestamp": dt_ts(),  # unix timestamp ms
+            "amount": 12.512,
+            "cost": 1000,
+            "fee": None,
+            "fees": [],
+            "id": "354669640",
+            "order": None,
+            "price": 613.84,
+            "side": "buy",
+            "takerOrMaker": None,
+            "type": None,
+        },
+    ]
+
+    caplog.set_level(logging.DEBUG)
+    exchange = get_patched_exchange(mocker, default_conf)
+    exchange._api_async.fetch_trades = get_mock_coro(trades)
+    exchange._ft_has["exchange_has_overrides"]["fetchTrades"] = True
+
+    pairs = [("IOTA/USDT:USDT", "5m", candle_type),
+             ("XRP/USDT:USDT", "5m", candle_type)]
+    # empty dicts
+    assert not exchange._trades
+    res = exchange.refresh_latest_trades(pairs, cache=False)
+    # No caching
+    assert not exchange._trades
+
+    assert len(res) == len(pairs)
+    assert exchange._api_async.fetch_trades.call_count == 4
+    exchange._api_async.fetch_trades.reset_mock()
+
+    exchange.required_candle_call_count = 2
+    res = exchange.refresh_latest_trades(pairs)
+    assert len(res) == len(pairs)
+
+    assert log_has(f"Refreshing TRADES data for {len(pairs)} pairs", caplog)
+    assert exchange._trades
+    assert exchange._api_async.fetch_trades.call_count == 4
+    exchange._api_async.fetch_trades.reset_mock()
+    for pair in pairs:
+        assert isinstance(exchange.trades(pair), DataFrame)
+        assert len(exchange.trades(pair)) > 0
+
+        # trades function should return a different object on each call
+        # if copy is "True"
+        assert exchange.trades(pair) is not exchange.trades(pair)
+        assert exchange.trades(pair) is not exchange.trades(pair, copy=True)
+        assert exchange.trades(
+            pair, copy=True) is not exchange.trades(pair, copy=True)
+        assert exchange.trades(
+            pair, copy=False) is exchange.trades(pair, copy=False)
+
+        # test caching
+        ohlcv = [
+            [
+                dt_ts(dt_now() - timedelta(minutes=5)),  # unix timestamp ms
+                1,  # open
+                2,  # high
+                3,  # low
+                4,  # close
+                5,  # volume (in quote currency)
+            ],
+            [
+                dt_ts(),  # unix timestamp ms
+                3,  # open
+                1,  # high
+                4,  # low
+                6,  # close
+                5,  # volume (in quote currency)
+            ],
+        ]
+        cols = DEFAULT_DATAFRAME_COLUMNS
+        trades_df = DataFrame(ohlcv, columns=cols)
+
+        trades_df["date"] = to_datetime(trades_df["date"], unit="ms", utc=True)
+        trades_df["date"] = trades_df["date"].apply(
+            lambda date: timeframe_to_prev_date("5m", date))
+        exchange._klines[pair] = trades_df
+    res = exchange.refresh_latest_trades(
+        [("IOTA/USDT:USDT", "5m", candle_type),
+         ("XRP/USDT:USDT", "5m", candle_type)]
+    )
+    assert len(res) == 0
+    assert exchange._api_async.fetch_trades.call_count == 0
+    caplog.clear()
+
+    # Reset refresh times
+    for pair in pairs:
+        # test caching with "expired" candle
+        trades = [
+            {
+                # unix timestamp ms
+                "timestamp": dt_ts(exchange._klines[pair].iloc[-1].date - timedelta(minutes=5)),
+                "amount": 16.512,
+                "cost": 10134.07488,
+                "fee": None,
+                "fees": [],
+                "id": "354669639",
+                "order": None,
+                "price": 613.74,
+                "side": "sell",
+                "takerOrMaker": None,
+                "type": None,
+            }
+        ]
+        trades_df = DataFrame(trades)
+        trades_df["date"] = to_datetime(
+            trades_df["timestamp"], unit="ms", utc=True)
+        exchange._trades[pair] = trades_df
+    res = exchange.refresh_latest_trades(
+        [("IOTA/USDT:USDT", "5m", candle_type),
+         ("XRP/USDT:USDT", "5m", candle_type)]
+    )
+    assert len(res) == len(pairs)
+
+    assert exchange._api_async.fetch_trades.call_count == 4
+
+    # cache - but disabled caching
+    exchange._api_async.fetch_trades.reset_mock()
+    exchange.required_candle_call_count = 1
+
+    pairlist = [
+        ("IOTA/ETH", "5m", candle_type),
+        ("XRP/ETH", "5m", candle_type),
+        ("XRP/ETH", "1d", candle_type),
+    ]
+    res = exchange.refresh_latest_trades(pairlist, cache=False)
+    assert len(res) == 3
+    assert exchange._api_async.fetch_trades.call_count == 6
+
+    # Test the same again, should NOT return from cache!
+    exchange._api_async.fetch_trades.reset_mock()
+    res = exchange.refresh_latest_trades(pairlist, cache=False)
+    assert len(res) == 3
+    assert exchange._api_async.fetch_trades.call_count == 6
+    exchange._api_async.fetch_trades.reset_mock()
+    caplog.clear()
+
+
 @pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT])
 def test_refresh_latest_ohlcv_cache(mocker, default_conf, candle_type, time_machine) -> None:
     start = datetime(2021, 8, 1, 0, 0, 0, 0, tzinfo=timezone.utc)