From e49b5b03dba5ef46c13c3c63054208762f8ec2f2 Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Wed, 30 Oct 2024 07:59:06 +0800 Subject: [PATCH] feat: stop on 404 to prevent missing data --- freqtrade/exchange/binance.py | 1 + freqtrade/exchange/binance_public_data.py | 28 +++++++++++++++++----- tests/exchange/test_binance_public_data.py | 21 ++++++++++++---- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 86accf055..65baf7c2c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -142,6 +142,7 @@ class Binance(Exchange): timeframe=timeframe, since_ms=since_ms, until_ms=until_ms, + stop_on_404=True, ) ) if df.empty: diff --git a/freqtrade/exchange/binance_public_data.py b/freqtrade/exchange/binance_public_data.py index b2b449510..ba9d2f2e9 100644 --- a/freqtrade/exchange/binance_public_data.py +++ b/freqtrade/exchange/binance_public_data.py @@ -27,12 +27,18 @@ class BadHttpStatus(Exception): async def fetch_ohlcv( - candle_type: CandleType, pair: str, timeframe: str, since_ms: int, until_ms: int | None + candle_type: CandleType, + pair: str, + timeframe: str, + since_ms: int, + until_ms: int | None, + stop_on_404: bool = False, ) -> DataFrame: """ Fetch OHLCV data from https://data.binance.vision/ :candle_type: Currently only spot and futures are supported :param until_ms: `None` indicates the timestamp of the latest available data + :param stop_on_404: Stop to download the following data when a 404 returned :return: None if no data available in the time range """ if candle_type == CandleType.SPOT: @@ -51,7 +57,7 @@ async def fetch_ohlcv( end = min(end, last_available_date) if start >= end: return DataFrame() - return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end) + return await _fetch_ohlcv(asset_type, symbol, timeframe, start, end, stop_on_404) def symbol_ccxt_to_binance(symbol: str) -> str: @@ -60,7 +66,7 @@ def symbol_ccxt_to_binance(symbol: str) -> str: e.g. BTC/USDT -> BTCUSDT, BTC/USDT:USDT -> BTCUSDT """ if ":" in symbol: - parts = symbol.split() + parts = symbol.split(":") if len(parts) != 2: raise ValueError(f"Cannot recognize symbol: {symbol}") return parts[0].replace("/", "") @@ -75,7 +81,14 @@ def concat(dfs) -> DataFrame: return pd.concat(dfs) -async def _fetch_ohlcv(asset_type, symbol, timeframe, start, end) -> DataFrame: +async def _fetch_ohlcv( + asset_type: str, + symbol: str, + timeframe: str, + start: datetime.date, + end: datetime.date, + stop_on_404: bool, +) -> DataFrame: dfs: list[DataFrame | None] = [] connector = aiohttp.TCPConnector(limit=100) @@ -93,6 +106,9 @@ async def _fetch_ohlcv(asset_type, symbol, timeframe, start, end) -> DataFrame: # Directly return the existing data, do not allow the gap # between the data return concat(dfs) + elif result is None and stop_on_404: + logger.debug("Abort downloading from data.binance.vision due to 404") + return concat(dfs) else: dfs.append(result) return concat(dfs) @@ -175,12 +191,12 @@ async def get_daily_ohlcv( df["date"] = pd.to_datetime(df["date"], unit="ms", utc=True) return df elif resp.status == 404: - logger.warning(f"No data available for {symbol} in {format_date(date)}") + logger.debug(f"No data available for {symbol} in {format_date(date)}") return None else: raise BadHttpStatus(f"{resp.status} - {resp.reason}") except Exception as e: retry += 1 if retry >= retry_count: - logger.warning(f"Failed to get data from {url}: {e}") + logger.debug(f"Failed to get data from {url}: {e}") raise diff --git a/tests/exchange/test_binance_public_data.py b/tests/exchange/test_binance_public_data.py index 5e0a414f2..995f562a7 100644 --- a/tests/exchange/test_binance_public_data.py +++ b/tests/exchange/test_binance_public_data.py @@ -93,48 +93,61 @@ def make_response_from_url(start_date, end_date): @pytest.mark.parametrize( - "since,until,first_date,last_date", + "since,until,first_date,last_date,stop_on_404", [ - (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23)), + (dt_utc(2020, 1, 1), dt_utc(2020, 1, 2), dt_utc(2020, 1, 1), dt_utc(2020, 1, 2, 23), False), ( dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23, 59, 59), dt_utc(2020, 1, 1), dt_utc(2020, 1, 1, 23), + False, ), ( dt_utc(2020, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), dt_utc(2020, 1, 3, 23), + False, ), ( dt_utc(2019, 1, 1), dt_utc(2020, 1, 5), dt_utc(2020, 1, 1), dt_utc(2020, 1, 3, 23), + False, ), ( dt_utc(2019, 1, 1), dt_utc(2019, 1, 5), None, None, + False, ), ( dt_utc(2021, 1, 1), dt_utc(2021, 1, 5), None, None, + False, ), ( dt_utc(2020, 1, 2), None, dt_utc(2020, 1, 2), dt_utc(2020, 1, 3, 23), + False, + ), + ( + dt_utc(2019, 1, 1), + dt_utc(2020, 1, 5), + None, + None, + True, ), ], ) -async def test_fetch_ohlcv(mocker, since, until, first_date, last_date): +async def test_fetch_ohlcv(mocker, since, until, first_date, last_date, stop_on_404): history_start = dt_utc(2020, 1, 1).date() history_end = dt_utc(2020, 1, 3).date() candle_type = CandleType.SPOT @@ -147,7 +160,7 @@ async def test_fetch_ohlcv(mocker, since, until, first_date, last_date): mocker.patch( "aiohttp.ClientSession.get", side_effect=make_response_from_url(history_start, history_end) ) - df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms) + df = await fetch_ohlcv(candle_type, pair, timeframe, since_ms, until_ms, stop_on_404) if df.empty: assert first_date is None and last_date is None