diff --git a/freqtrade/data/history/datahandlers/featherdatahandler.py b/freqtrade/data/history/datahandlers/featherdatahandler.py index 41978bb13..6e813c046 100644 --- a/freqtrade/data/history/datahandlers/featherdatahandler.py +++ b/freqtrade/data/history/datahandlers/featherdatahandler.py @@ -143,7 +143,7 @@ class FeatherDataHandler(IDataHandler): except (ImportError, AttributeError, ValueError) as e: # Fallback: load entire file - logger.debug(f"Unable to use Arrow filtering, loading entire trades file: {e}") + logger.warning(f"Unable to use Arrow filtering, loading entire trades file: {e}") tradesdata = read_feather(filename) return tradesdata diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 140bd7e0c..30c5f0590 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -41,7 +41,7 @@ from freqtrade.strategy.informative_decorator import ( ) from freqtrade.strategy.strategy_validation import StrategyResultValidator from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper -from freqtrade.util import dt_now +from freqtrade.util import dt_now, dt_ts from freqtrade.wallets import Wallets @@ -1770,29 +1770,14 @@ class IStrategy(ABC, HyperStrategyMixin): pair = metadata["pair"] # Build timerange from dataframe date column if not dataframe.empty: - start_ts = int(dataframe["date"].iloc[0].timestamp() * 1000) - end_ts = int(dataframe["date"].iloc[-1].timestamp() * 1000) + start_ts = dt_ts(dataframe["date"].iloc[0]) + end_ts = dt_ts(dataframe["date"].iloc[-1]) timerange = TimeRange("date", "date", startts=start_ts, stopts=end_ts) else: timerange = None trades = self.dp.trades(pair=pair, copy=False, timerange=timerange) - # Apply additional filtering with buffer for faster backtesting - if not trades.empty and not dataframe.empty and "timestamp" in trades.columns: - # Add timeframe buffer to ensure complete candle coverage - timeframe_buffer = timeframe_to_seconds(self.config["timeframe"]) * 1000 - - # Create time bounds with buffer - time_start = start_ts - timeframe_buffer - time_end = end_ts + timeframe_buffer - - # Filter trades within buffered timerange - trades_mask = (trades["timestamp"] >= time_start) & ( - trades["timestamp"] <= time_end - ) - trades = trades.loc[trades_mask].reset_index(drop=True) - cached_grouped_trades: DataFrame | None = self._cached_grouped_trades_per_pair.get(pair) dataframe, cached_grouped_trades = populate_dataframe_with_trades( cached_grouped_trades, self.config, dataframe, trades diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 98c7c65ba..6d6cc40ae 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -506,3 +506,70 @@ def test_get_datahandler(testdatadir): assert isinstance(dh, JsonGzDataHandler) dh1 = get_datahandler(testdatadir, "jsongz", dh) assert id(dh1) == id(dh) + + +@pytest.fixture +def feather_dh(testdatadir): + return FeatherDataHandler(testdatadir) + + +@pytest.fixture +def trades_full(feather_dh): + df = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT) + assert not df.empty + return df + + +@pytest.fixture +def timerange_full(trades_full): + # Pick a full-span window using actual timestamps + startts = int(trades_full["timestamp"].min()) + stopts = int(trades_full["timestamp"].max()) + return TimeRange("date", "date", startts=startts, stopts=stopts) + + +@pytest.fixture +def timerange_mid(trades_full): + # Pick a mid-range window using actual timestamps + mid_start = int(trades_full["timestamp"].iloc[len(trades_full) // 3]) + mid_end = int(trades_full["timestamp"].iloc[(2 * len(trades_full)) // 3]) + return TimeRange("date", "date", startts=mid_start, stopts=mid_end) + + +def test_feather_trades_timerange_filter_fullspan(feather_dh, trades_full, timerange_full): + # Full-span filter should equal unfiltered + filtered = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_full) + assert_frame_equal( + trades_full.reset_index(drop=True), filtered.reset_index(drop=True), check_exact=True + ) + + +def test_feather_trades_timerange_filter_subset(feather_dh, trades_full, timerange_mid): + # Subset filter should be a subset of the full-span filter + subset = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_mid) + assert not subset.empty + assert subset["timestamp"].min() >= timerange_mid.startts + assert subset["timestamp"].max() <= timerange_mid.stopts + assert len(subset) < len(trades_full) + + +def test_feather_trades_timerange_pushdown_fallback( + feather_dh, trades_full, timerange_mid, monkeypatch, caplog +): + # Pushdown filter should fail, so fallback should load the entire file + import freqtrade.data.history.datahandlers.featherdatahandler as fdh + + def raise_err(*args, **kwargs): + raise ValueError("fail") + + # Mock the dataset loading to raise an error + monkeypatch.setattr(fdh.dataset, "dataset", raise_err) + + with caplog.at_level("WARNING"): + out = feather_dh.trades_load("XRP/ETH", TradingMode.SPOT, timerange=timerange_mid) + + assert len(out) == len(trades_full) + assert any( + "Unable to use Arrow filtering, loading entire trades file" in r.message + for r in caplog.records + )