Merge pull request #11289 from freqtrade/feat/binance_trades_fast

Binance: Download trades "fast" from binance.vision

tests/exchange/test_binance.py

@@ -6,6 +6,7 @@ import ccxt
 import pandas as pd
 import pytest
 
+from freqtrade.data.converter.trade_converter import trades_dict_to_list
 from freqtrade.enums import CandleType, MarginMode, TradingMode
 from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException
 from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_seconds
@@ -1002,6 +1003,7 @@ def test_get_maintenance_ratio_and_amt_binance(
 
 
 async def test__async_get_trade_history_id_binance(default_conf_usdt, mocker, fetch_trades_result):
+    default_conf_usdt["exchange"]["only_from_ccxt"] = True
     exchange = get_patched_exchange(mocker, default_conf_usdt, exchange="binance")
 
     async def mock_get_trade_hist(pair, *args, **kwargs):
@@ -1056,3 +1058,53 @@ async def test__async_get_trade_history_id_binance(default_conf_usdt, mocker, fe
 
     # Clean up event loop to avoid warnings
     exchange.close()
+
+
+async def test__async_get_trade_history_id_binance_fast(
+    default_conf_usdt, mocker, fetch_trades_result
+):
+    default_conf_usdt["exchange"]["only_from_ccxt"] = False
+    exchange = get_patched_exchange(mocker, default_conf_usdt, exchange="binance")
+
+    async def mock_get_trade_hist(pair, *args, **kwargs):
+        if "since" in kwargs:
+            pass
+            # older than initial call
+            # if kwargs["since"] < 1565798399752:
+            #     return []
+            # else:
+            #     # Don't expect to get here
+            #     raise ValueError("Unexpected call")
+            #     # return fetch_trades_result[:-2]
+        elif kwargs.get("params", {}).get(exchange._trades_pagination_arg) == "0":
+            # Return first 3
+            return fetch_trades_result[:-2]
+        # elif kwargs.get("params", {}).get(exchange._trades_pagination_arg) in (
+        #     fetch_trades_result[-3]["id"],
+        #     1565798399752,
+        # ):
+        #     # Return 2
+        #     return fetch_trades_result[-3:-1]
+        # else:
+        #     # Return last 2
+        #     return fetch_trades_result[-2:]
+
+    pair = "ETH/BTC"
+    mocker.patch(
+        "freqtrade.exchange.binance.download_archive_trades",
+        return_value=(pair, trades_dict_to_list(fetch_trades_result[-2:])),
+    )
+
+    exchange._api_async.fetch_trades = MagicMock(side_effect=mock_get_trade_hist)
+
+    ret = await exchange._async_get_trade_history(
+        pair,
+        since=fetch_trades_result[0]["timestamp"],
+        until=fetch_trades_result[-1]["timestamp"] - 1,
+    )
+
+    assert ret[0] == pair
+    assert isinstance(ret[1], list)
+
+    # Clean up event loop to avoid warnings
+    exchange.close()

tests/exchange/test_binance_public_data.py

@@ -14,11 +14,15 @@ from freqtrade.enums import CandleType
 from freqtrade.exchange.binance_public_data import (
     BadHttpStatus,
     Http404,
+    binance_vision_trades_zip_url,
     binance_vision_zip_name,
     download_archive_ohlcv,
+    download_archive_trades,
     get_daily_ohlcv,
+    get_daily_trades,
 )
 from freqtrade.util.datetime_helpers import dt_ts, dt_utc
+from ft_client.test_client.test_rest_client import log_has_re
 
 
 @pytest.fixture(scope="module")
@@ -337,3 +341,156 @@ async def test_get_daily_ohlcv(mocker, testdatadir):
         with pytest.raises(zipfile.BadZipFile):
             df = await get_daily_ohlcv(symbol, timeframe, CandleType.SPOT, date, session)
         assert get.call_count == 4  # 1 + 3 default retries
+
+
+async def test_download_archive_trades(mocker, caplog):
+    pair = "BTC/USDT"
+
+    since_ms = dt_ts(dt_utc(2020, 1, 1))
+    until_ms = dt_ts(dt_utc(2020, 1, 2))
+    markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}}
+
+    mocker.patch("freqtrade.exchange.binance_public_data.get_daily_trades", return_value=[[2, 3]])
+
+    pair1, res = await download_archive_trades(
+        CandleType.SPOT, pair, since_ms=since_ms, until_ms=until_ms, markets=markets
+    )
+    assert pair1 == pair
+    assert res == [[2, 3], [2, 3]]
+
+    mocker.patch(
+        "freqtrade.exchange.binance_public_data.get_daily_trades",
+        side_effect=Http404("xxx", dt_utc(2020, 1, 1), "http://example.com/something"),
+    )
+
+    pair1, res = await download_archive_trades(
+        CandleType.SPOT, pair, since_ms=since_ms, until_ms=until_ms, markets=markets
+    )
+
+    assert pair1 == pair
+    assert res == []
+    # exit on day 1
+    assert log_has_re("Fast download is unavailable", caplog)
+
+    # Test fail on day 2
+    caplog.clear()
+    mocker.patch(
+        "freqtrade.exchange.binance_public_data.get_daily_trades",
+        side_effect=[
+            [[2, 3]],
+            [[2, 3]],
+            Http404("xxx", dt_utc(2020, 1, 2), "http://example.com/something"),
+            [[2, 3]],
+        ],
+    )
+    # Download 3 days
+    until_ms = dt_ts(dt_utc(2020, 1, 3))
+
+    pair1, res = await download_archive_trades(
+        CandleType.SPOT, pair, since_ms=since_ms, until_ms=until_ms, markets=markets
+    )
+
+    assert pair1 == pair
+    assert res == [[2, 3], [2, 3]]
+    assert log_has_re(r"Binance fast download .*stopped", caplog)
+
+
+async def test_download_archive_trades_exception(mocker, caplog):
+    pair = "BTC/USDT"
+
+    since_ms = dt_ts(dt_utc(2020, 1, 1))
+    until_ms = dt_ts(dt_utc(2020, 1, 2))
+
+    markets = {"BTC/USDT": {"id": "BTCUSDT"}, "BTC/USDT:USDT": {"id": "BTCUSDT"}}
+    mocker.patch(
+        "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get", side_effect=RuntimeError
+    )
+
+    pair1, res = await download_archive_trades(
+        CandleType.SPOT, pair, since_ms=since_ms, until_ms=until_ms, markets=markets
+    )
+
+    assert pair1 == pair
+    assert res == []
+    mocker.patch(
+        "freqtrade.exchange.binance_public_data._download_archive_trades", side_effect=RuntimeError
+    )
+
+    await download_archive_trades(
+        CandleType.SPOT, pair, since_ms=since_ms, until_ms=until_ms, markets=markets
+    )
+    assert pair1 == pair
+    assert res == []
+    assert log_has_re("An exception occurred during fast trades download", caplog)
+
+
+async def test_binance_vision_trades_zip_url():
+    url = binance_vision_trades_zip_url("BTCUSDT", CandleType.SPOT, dt_utc(2023, 10, 27))
+    assert (
+        url == "https://data.binance.vision/data/spot/daily/aggTrades/"
+        "BTCUSDT/BTCUSDT-aggTrades-2023-10-27.zip"
+    )
+
+    url = binance_vision_trades_zip_url("BTCUSDT", CandleType.FUTURES, dt_utc(2023, 10, 28))
+    assert (
+        url == "https://data.binance.vision/data/futures/um/daily/aggTrades/"
+        "BTCUSDT/BTCUSDT-aggTrades-2023-10-28.zip"
+    )
+
+
+async def test_get_daily_trades(mocker, testdatadir):
+    symbol = "PEPEUSDT"
+    symbol_futures = "APEUSDT"
+    date = dt_utc(2024, 10, 28).date()
+    first_date = 1729987202368
+    last_date = 1730073596350
+
+    async with aiohttp.ClientSession() as session:
+        spot_path = (
+            testdatadir / "binance/binance_public_data/spot-PEPEUSDT-aggTrades-2024-10-27.zip"
+        )
+        get = mocker.patch(
+            "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get",
+            return_value=MockResponse(spot_path.read_bytes(), 200),
+        )
+        res = await get_daily_trades(symbol, CandleType.SPOT, date, session)
+        assert get.call_count == 1
+        assert res[0][0] == first_date
+        assert res[-1][0] == last_date
+
+        futures_path = (
+            testdatadir / "binance/binance_public_data/futures-APEUSDT-aggTrades-2024-10-18.zip"
+        )
+        get = mocker.patch(
+            "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get",
+            return_value=MockResponse(futures_path.read_bytes(), 200),
+        )
+        res_fut = await get_daily_trades(symbol_futures, CandleType.FUTURES, date, session)
+        assert get.call_count == 1
+        assert res_fut[0][0] == 1729209603958
+        assert res_fut[-1][0] == 1729295981272
+
+        get = mocker.patch(
+            "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get",
+            return_value=MockResponse(b"", 404),
+        )
+        with pytest.raises(Http404):
+            await get_daily_trades(symbol, CandleType.SPOT, date, session, retry_delay=0)
+        assert get.call_count == 1
+
+        get = mocker.patch(
+            "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get",
+            return_value=MockResponse(b"", 500),
+        )
+        mocker.patch("asyncio.sleep")
+        with pytest.raises(BadHttpStatus):
+            await get_daily_trades(symbol, CandleType.SPOT, date, session)
+        assert get.call_count == 4  # 1 + 3 default retries
+
+        get = mocker.patch(
+            "freqtrade.exchange.binance_public_data.aiohttp.ClientSession.get",
+            return_value=MockResponse(b"nop", 200),
+        )
+        with pytest.raises(zipfile.BadZipFile):
+            await get_daily_trades(symbol, CandleType.SPOT, date, session)
+        assert get.call_count == 4  # 1 + 3 default retries

tests/exchange/test_exchange.py

@@ -2373,6 +2373,8 @@ def test_refresh_latest_trades(
     caplog.set_level(logging.DEBUG)
     use_trades_conf = default_conf
     use_trades_conf["exchange"]["use_public_trades"] = True
+    use_trades_conf["exchange"]["only_from_ccxt"] = True
+
     use_trades_conf["datadir"] = tmp_path
     use_trades_conf["orderflow"] = {"max_candles": 1500}
     exchange = get_patched_exchange(mocker, use_trades_conf)
@@ -3365,6 +3367,7 @@ async def test__async_fetch_trades_contract_size(
 async def test__async_get_trade_history_id(
     default_conf, mocker, exchange_name, fetch_trades_result
 ):
+    default_conf["exchange"]["only_from_ccxt"] = True
     exchange = get_patched_exchange(mocker, default_conf, exchange=exchange_name)
     if exchange._trades_pagination != "id":
         exchange.close()