diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 65baf7c2c..1b227265a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -142,14 +142,13 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, - stop_on_404=True, + stop_on_404=False, ) ) if df.empty: rest_since_ms = since_ms else: rest_since_ms = dt_ts(df.iloc[-1].date) + timeframe_to_msecs(timeframe) - if until_ms and rest_since_ms > until_ms: rest_df = DataFrame() else: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index ba9d2f2e9..007886606 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -39,7 +39,8 @@ async def fetch_ohlcv( :candle_type: Currently only spot and futures are supported :param until_ms: `None` indicates the timestamp of the latest available data :param stop_on_404: Stop to download the following data when a 404 returned - :return: None if no data available in the time range + :return: the date range is between [since_ms, until_ms), + return None if no data available in the time range """ if candle_type == CandleType.SPOT: asset_type = "spot" @@ -57,7 +58,14 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + df = await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) + logger.info( + f"Downloaded data for {pair} from https://data.binance.vision/ with length {len(df)}." + ) + if not df.empty: + return df.loc[(df["date"] >= start) & (df["date"] < end)] + else: + return df def symbol_ccxt_to_binance(symbol: str) -> str: @@ -149,7 +157,7 @@ async def get_daily_ohlcv( date: datetime.date, session: aiohttp.ClientSession, retry_count: int = 3, -) -> DataFrame | None: +) -> DataFrame | None | Exception: """ Get daily OHLCV from https://data.binance.vision See https://github.com/binance/binance-public-data @@ -199,4 +207,4 @@ async def get_daily_ohlcv( retry += 1 if retry >= retry_count: logger.debug(f"Failed to get data from {url}: {e}") - raise + return e diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 48bb84369..5e1e32a54 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2246,7 +2246,7 @@ class Exchange: candle_type=candle_type, ) ) - logger.info(f"Downloaded data for {pair} with length {len(data)}.") + logger.info(f"Downloaded data for {pair} from ccxt with length {len(data)}.") return ohlcv_to_dataframe(data, timeframe, pair, fill_missing=False, drop_incomplete=True) async def _async_get_historic_ohlcv( diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 4dfc4d6db..749dff663 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from random import randint from unittest.mock import AsyncMock, MagicMock, PropertyMock @@ -8,9 +8,10 @@ import pytest from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException +from freqtrade.exchange.exchange_utils_timeframe import timeframe_to_seconds from freqtrade.persistence import Trade from freqtrade.util.datetime_helpers import dt_from_ts, dt_ts, dt_utc -from tests.conftest import EXMS, get_mock_coro, get_patched_exchange, log_has_re +from tests.conftest import EXMS, get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -733,17 +734,17 @@ def test__set_leverage_binance(mocker, default_conf): ) -def make_storage(start: datetime, end: datetime, timeframe: str = "1min"): - date = pd.date_range(start, end, freq=timeframe) +def make_storage(start: datetime, end: datetime, timeframe: str): + date = pd.date_range(start, end, freq=timeframe.replace("m", "min")) df = pd.DataFrame( data=dict(date=date, open=1.0, high=1.0, low=1.0, close=1.0), ) return df -def patch_ohlcv(mocker, start, archive_end, api_end): - archive_storage = make_storage(start, archive_end) - api_storage = make_storage(start, api_end) +def patch_ohlcv(mocker, start, archive_end, api_end, timeframe): + archive_storage = make_storage(start, archive_end, timeframe) + api_storage = make_storage(start, api_end, timeframe) ohlcv = [[dt_ts(start), 1, 1, 1, 1]] # (pair, timeframe, candle_type, ohlcv, True) @@ -768,6 +769,7 @@ def patch_ohlcv(mocker, start, archive_end, api_end): timeframe, since_ms, until_ms, + stop_on_404=False, ): since = dt_from_ts(since_ms) until = dt_from_ts(until_ms) if until_ms else archive_end + timedelta(seconds=1) @@ -909,11 +911,22 @@ def patch_ohlcv(mocker, start, archive_end, api_end): dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), - dt_utc(2020, 1, 1, 23, 59), + dt_utc(2020, 1, 1, 23), False, False, True, ), + ( + "1m", + False, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 3, 50, 30), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 3, 50), + False, + True, + False, + ), ], ) def test_get_historic_ohlcv_binance( @@ -935,7 +948,7 @@ def test_get_historic_ohlcv_binance( archive_end = dt_utc(2020, 1, 2) api_end = dt_utc(2020, 1, 3) candle_mock, api_mock, archive_mock = patch_ohlcv( - mocker, start=start, archive_end=archive_end, api_end=api_end + mocker, start=start, archive_end=archive_end, api_end=api_end, timeframe=timeframe ) candle_type = CandleType.SPOT @@ -952,6 +965,9 @@ def test_get_historic_ohlcv_binance( else: assert df["date"].iloc[0] == first_date assert df["date"].iloc[-1] == last_date + assert ( + df["date"].diff().iloc[1:] == timedelta(seconds=timeframe_to_seconds(timeframe)) + ).all() if candle_called: candle_mock.assert_called_once() @@ -961,45 +977,6 @@ def test_get_historic_ohlcv_binance( api_mock.assert_called_once() -@pytest.mark.xfail(reason="Need refactor") -@pytest.mark.parametrize("candle_type", [CandleType.MARK, ""]) -async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, candle_type): - ohlcv = [ - [ - int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - - exchange = get_patched_exchange(mocker, default_conf, exchange="binance") - # Monkey-patch async function - exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) - - pair = "ETH/BTC" - respair, restf, restype, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=False, candle_type=candle_type - ) - assert respair == pair - assert restf == "5m" - assert restype == candle_type - # Call with very old timestamp - causes tons of requests - assert exchange._api_async.fetch_ohlcv.call_count > 400 - # assert res == ohlcv - exchange._api_async.fetch_ohlcv.reset_mock() - _, _, _, res, _ = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=True, candle_type=candle_type - ) - - # Called twice - one "init" call - and one to get the actual data. - assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert res == ohlcv - assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog) - - @pytest.mark.parametrize( "pair,notional_value,mm_ratio,amt", [ diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 995f562a7..f598a0bdd 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -13,6 +13,7 @@ from freqtrade.exchange.binance_public_data import ( BadHttpStatus, fetch_ohlcv, get_daily_ohlcv, + symbol_ccxt_to_binance, zip_name, ) from freqtrade.util.datetime_helpers import dt_ts, dt_utc @@ -93,10 +94,18 @@ def make_response_from_url(start_date, end_date): @pytest.mark.parametrize( - "since,until,first_date,last_date,stop_on_404", + "candle_type,since,until,first_date,last_date,stop_on_404", [ - (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23), False), ( + CandleType.SPOT, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 2), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23), + False, + ), + ( + CandleType.SPOT, dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), @@ -104,6 +113,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2020, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -111,6 +121,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), @@ -118,6 +129,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2019, 1, 5), None, @@ -125,6 +137,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2021, 1, 1), dt_utc(2021, 1, 5), None, @@ -132,6 +145,7 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2020, 1, 2), None, dt_utc(2020, 1, 2), @@ -139,20 +153,44 @@ def make_response_from_url(start_date, end_date): False, ), ( + CandleType.SPOT, dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), None, None, True, ), + ( + CandleType.SPOT, + dt_utc(2020, 1, 5), + dt_utc(2020, 1, 1), + None, + None, + False, + ), + ( + CandleType.FUTURES, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59, 59), + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23), + False, + ), + ( + CandleType.INDEX, + dt_utc(2020, 1, 1), + dt_utc(2020, 1, 1, 23, 59, 59), + None, + None, + False, + ), ], ) -async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_404): +async def test_fetch_ohlcv(mocker, candle_type, since, until, first_date, last_date, stop_on_404): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() - candle_type = CandleType.SPOT - pair = "BTC/USDT" timeframe = "1h" + pair = "BTCUSDT" since_ms = dt_ts(since) until_ms = dt_ts(until) @@ -160,13 +198,32 @@ async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_ mocker.patch( "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) - if df.empty: - assert first_date is None and last_date is None + if candle_type in [CandleType.SPOT, CandleType.FUTURES]: + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + + if df.empty: + assert first_date is None and last_date is None + else: + assert df["date"].iloc[0] == first_date + assert df["date"].iloc[-1] == last_date else: - assert df["date"].iloc[0] == first_date - assert df["date"].iloc[-1] == last_date + with pytest.raises(ValueError): + await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) + + +async def test_fetch_ohlcv_exc(mocker): + timeframe = "1h" + pair = "BTCUSDT" + + since_ms = dt_ts(dt_utc(2020, 1, 1)) + until_ms = dt_ts(dt_utc(2020, 1, 2)) + + mocker.patch("aiohttp.ClientSession.get", side_effect=RuntimeError) + + df = await fetch_ohlcv(CandleType.SPOT, pair, timeframe, since_ms, until_ms) + + assert df.empty async def test_get_daily_ohlcv(mocker, testdatadir): @@ -197,9 +254,16 @@ async def test_get_daily_ohlcv(mocker, testdatadir): mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"", 500)) mocker.patch("asyncio.sleep") - with pytest.raises(BadHttpStatus): - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert isinstance(df, BadHttpStatus) mocker.patch("aiohttp.ClientSession.get", return_value=MockResponse(b"nop", 200)) - with pytest.raises(zipfile.BadZipFile): - df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + df = await get_daily_ohlcv("spot", symbol, timeframe, date, session) + assert isinstance(df, zipfile.BadZipFile) + + +def test_symbol_ccxt_to_binance(): + assert symbol_ccxt_to_binance("BTC/USDT") == "BTCUSDT" + assert symbol_ccxt_to_binance("BTC/USDT:USDT") == "BTCUSDT" + with pytest.raises(ValueError): + assert symbol_ccxt_to_binance("BTC:USDT:USDT") == "BTCUSDT" diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 07d5927c2..b123229ad 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2124,7 +2124,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data after truncating the open candle. assert len(ret) == 2 - assert log_has_re(r"Downloaded data for .* with length .*\.", caplog) + assert log_has_re(r"Downloaded data for .* from ccxt with length .*\.", caplog) caplog.clear() diff --git a/tests/exchange_online/test_binance_compare_ohlcv.py b/tests/exchange_online/test_binance_compare_ohlcv.py index 7a76e19f2..af94b1bd1 100644 --- a/tests/exchange_online/test_binance_compare_ohlcv.py +++ b/tests/exchange_online/test_binance_compare_ohlcv.py @@ -1,10 +1,23 @@ """ Check if the earliest klines from rest API have its counterpart on https://data.binance.vision +Not expected to run in CI, manually run from shell: -Not expected to run in CI + TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py -Manually run from shell: -TEST_BINANCE_COMPARE_OHLCV=1 pytest tests/exchange_online/test_binance_compare_ohlcv.py +Until 2024-10-30, there are three usdt-m futures symbols "lack" data +All SPOT symbols are good. + +BTCUSDT-1m 113 days +ARCHIVE: 2019-12-31 00:00:00 │ 2024-10-30 02:51:00 │ 2541772 +API: 2019-09-08 17:57:00 │ 2024-10-30 03:11:00 │ 2704874 + +ETHUSDT 34 days +ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 +API: 2019-11-27 07:45:00 │ 2020-03-01 11:03:00 │ 136999 + +BCHUSDT 12 days +ARCHIVE: 2019-12-31 00:00:00 │ 2020-02-29 23:59:00 │ 87840 +API: 2019-12-19 08:57:00 │ 2020-03-01 06:55:00 │ 104999 """ import asyncio