diff --git a/freqtrade/data/btanalysis/__init__.py b/freqtrade/data/btanalysis/__init__.py index 48fdcd9b5..1889429f6 100644 --- a/freqtrade/data/btanalysis/__init__.py +++ b/freqtrade/data/btanalysis/__init__.py @@ -25,6 +25,7 @@ from .bt_fileutils import ( trade_list_to_dataframe, update_backtest_metadata, ) +from .historic_precision import get_tick_size_over_time from .trade_parallelism import ( analyze_trade_parallelism, evaluate_result_multi, diff --git a/freqtrade/data/btanalysis/historic_precision.py b/freqtrade/data/btanalysis/historic_precision.py new file mode 100644 index 000000000..c8aa4fbee --- /dev/null +++ b/freqtrade/data/btanalysis/historic_precision.py @@ -0,0 +1,27 @@ +from pandas import DataFrame, Series + + +def get_tick_size_over_time(candles: DataFrame) -> Series: + """ + Calculate the number of significant digits for candles over time. + It's using the Monthly maximum of the number of significant digits for each month. + :param candles: DataFrame with OHLCV data + :return: Series with the average number of significant digits for each month + """ + # count the number of significant digits for the open and close prices + for col in ["open", "high", "low", "close"]: + candles[f"{col}_count"] = ( + candles[col].round(14).astype(str).str.extract(r"\.(\d*[1-9])")[0].str.len() + ) + candles["max_count"] = candles[["open_count", "close_count", "high_count", "low_count"]].max( + axis=1 + ) + + candles1 = candles.set_index("date", drop=True) + # Group by month and calculate the average number of significant digits + monthly_count_avg1 = candles1["max_count"].resample("MS").max() + # monthly_open_count_avg + # convert monthly_open_count_avg from 5.0 to 0.00001, 4.0 to 0.0001, ... + monthly_open_count_avg = 1 / 10**monthly_count_avg1 + + return monthly_open_count_avg diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbc77604b..8d0073593 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,14 +9,18 @@ from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta -from numpy import nan -from pandas import DataFrame +from numpy import isnan, nan +from pandas import DataFrame, Series from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongShort from freqtrade.data import history -from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe +from freqtrade.data.btanalysis import ( + find_existing_backtest_stats, + get_tick_size_over_time, + trade_list_to_dataframe, +) from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.data.metrics import combined_dataframes_with_rel_mean @@ -35,7 +39,7 @@ from freqtrade.exchange import ( price_to_precision, timeframe_to_seconds, ) -from freqtrade.exchange.exchange import Exchange +from freqtrade.exchange.exchange import TICK_SIZE, Exchange from freqtrade.ft_types import ( BacktestContentType, BacktestContentTypeIcomplete, @@ -121,6 +125,7 @@ class Backtesting: self.order_id_counter: int = 0 config["dry_run"] = True + self.price_pair_prec: dict[str, Series] = {} self.run_ids: dict[str, str] = {} self.strategylist: list[IStrategy] = [] self.all_bt_content: dict[str, BacktestContentType] = {} @@ -189,7 +194,6 @@ class Backtesting: self.fee = max(fee for fee in fees if fee is not None) logger.info(f"Using fee {self.fee:.4%} - worst case fee from exchange (lowest tier).") self.precision_mode = self.exchange.precisionMode - self.precision_mode_price = self.exchange.precision_mode_price if self.config.get("freqai_backtest_live_models", False): from freqtrade.freqai.utils import get_timerange_backtest_live_models @@ -316,6 +320,11 @@ class Backtesting: self.progress.set_new_value(1) self._load_bt_data_detail() + self.price_pair_prec = {} + for pair in self.pairlists.whitelist: + if pair in data: + # Load price precision logic + self.price_pair_prec[pair] = get_tick_size_over_time(data[pair]) return data, self.timerange def _load_bt_data_detail(self) -> None: @@ -385,6 +394,22 @@ class Backtesting: else: self.futures_data = {} + def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float | None, int]: + """ + Get pair precision at that moment in time + :param pair: Pair to get precision for + :param current_time: Time to get precision for + :return: tuple of price precision, precision_mode_price for the pair at that given time. + """ + precision_series = self.price_pair_prec.get(pair) + if precision_series is not None: + precision = precision_series.asof(current_time) + + if not isnan(precision): + # Force tick size if we define the precision + return precision, TICK_SIZE + return self.exchange.get_precision_price(pair), self.exchange.precision_mode_price + def disable_database_use(self): disable_database_use(self.timeframe) @@ -793,7 +818,7 @@ class Backtesting: ) if rate is not None and rate != close_rate: close_rate = price_to_precision( - rate, trade.price_precision, self.precision_mode_price + rate, trade.price_precision, trade.precision_mode_price ) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately @@ -926,6 +951,7 @@ class Backtesting: trade: LocalTrade | None, order_type: str, price_precision: float | None, + precision_mode_price: int, ) -> tuple[float, float, float, float]: if order_type == "limit": new_rate = strategy_safe_wrapper( @@ -941,9 +967,7 @@ class Backtesting: # We can't place orders higher than current high (otherwise it'd be a stop limit entry) # which freqtrade does not support in live. if new_rate is not None and new_rate != propose_rate: - propose_rate = price_to_precision( - new_rate, price_precision, self.precision_mode_price - ) + propose_rate = price_to_precision(new_rate, price_precision, precision_mode_price) if direction == "short": propose_rate = max(propose_rate, row[LOW_IDX]) else: @@ -1036,7 +1060,7 @@ class Backtesting: pos_adjust = trade is not None and requested_rate is None stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0) - precision_price = self.exchange.get_precision_price(pair) + precision_price, precision_mode_price = self.get_pair_precision(pair, current_time) propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake( pair, @@ -1049,6 +1073,7 @@ class Backtesting: trade, order_type, precision_price, + precision_mode_price, ) # replace proposed rate if another rate was requested @@ -1124,7 +1149,7 @@ class Backtesting: amount_precision=precision_amount, price_precision=precision_price, precision_mode=self.precision_mode, - precision_mode_price=self.precision_mode_price, + precision_mode_price=precision_mode_price, contract_size=contract_size, orders=[], ) diff --git a/tests/data/test_historic_precision.py b/tests/data/test_historic_precision.py new file mode 100644 index 000000000..472218b8b --- /dev/null +++ b/tests/data/test_historic_precision.py @@ -0,0 +1,92 @@ +# pragma pylint: disable=missing-docstring, C0103 + +from datetime import timezone + +import pandas as pd +from numpy import nan +from pandas import DataFrame, Timestamp + +from freqtrade.data.btanalysis.historic_precision import get_tick_size_over_time + + +def test_get_tick_size_over_time(): + """ + Test the get_tick_size_over_time function with predefined data + """ + # Create test dataframe with different levels of precision + data = { + "date": [ + Timestamp("2020-01-01 00:00:00", tz=timezone.utc), + Timestamp("2020-01-02 00:00:00", tz=timezone.utc), + Timestamp("2020-01-03 00:00:00", tz=timezone.utc), + Timestamp("2020-01-15 00:00:00", tz=timezone.utc), + Timestamp("2020-01-16 00:00:00", tz=timezone.utc), + Timestamp("2020-01-31 00:00:00", tz=timezone.utc), + Timestamp("2020-02-01 00:00:00", tz=timezone.utc), + Timestamp("2020-02-15 00:00:00", tz=timezone.utc), + Timestamp("2020-03-15 00:00:00", tz=timezone.utc), + ], + "open": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34], + "high": [1.23457, 1.235, 1.24, 1.3, 1.23456, 1.235, 2.3457, 2.34, 2.34], + "low": [1.23455, 1.233, 1.22, 1.1, 1.23456, 1.233, 2.3455, 2.34, 2.34], + "close": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34], + "volume": [100, 200, 300, 400, 500, 600, 700, 800, 900], + } + + candles = DataFrame(data) + + # Calculate significant digits + result = get_tick_size_over_time(candles) + + # Check that the result is a pandas Series + assert isinstance(result, pd.Series) + + # Check that we have three months of data (Jan, Feb and March 2020 ) + assert len(result) == 3 + + # Before + assert result.asof("2019-01-01 00:00:00+00:00") is nan + # January should have 5 significant digits (based on 1.23456789 being the most precise value) + # which should be converted to 0.00001 + + assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001 + assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001 + assert result.asof("2020-02-25 00:00:00+00:00") == 0.0001 + assert result.asof("2020-03-25 00:00:00+00:00") == 0.01 + assert result.asof("2020-04-01 00:00:00+00:00") == 0.01 + # Value far past the last date should be the last value + assert result.asof("2025-04-01 00:00:00+00:00") == 0.01 + + assert result.iloc[0] == 0.00001 + + +def test_get_tick_size_over_time_real_data(testdatadir): + """ + Test the get_tick_size_over_time function with real data from the testdatadir + """ + from freqtrade.data.history import load_pair_history + + # Load some test data from the testdata directory + pair = "UNITTEST/BTC" + timeframe = "1m" + + candles = load_pair_history( + datadir=testdatadir, + pair=pair, + timeframe=timeframe, + ) + + # Make sure we have test data + assert not candles.empty, "No test data found, cannot run test" + + # Calculate significant digits + result = get_tick_size_over_time(candles) + + assert isinstance(result, pd.Series) + + # Verify that all values are between 0 and 1 (valid precision values) + assert all(result > 0) + assert all(result < 1) + + assert all(result <= 0.0001) + assert all(result >= 0.00000001) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2973a4a49..9ea892ab8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,6 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import CandleType, ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_next_date, timeframe_to_prev_date +from freqtrade.exchange.exchange_utils import DECIMAL_PLACES, TICK_SIZE from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade @@ -348,6 +349,29 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: assert processed["UNITTEST/BTC"].equals(processed2["UNITTEST/BTC"]) +def test_get_pair_precision_bt(default_conf, mocker) -> None: + patch_exchange(mocker) + default_conf["timeframe"] = "30m" + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = "UNITTEST/BTC" + backtesting.pairlists._whitelist = [pair] + ex_mock = mocker.patch(f"{EXMS}.get_precision_price", return_value=1e-5) + data, timerange = backtesting.load_bt_data() + assert data + + assert backtesting.get_pair_precision(pair, dt_utc(2018, 1, 1)) == (1e-8, TICK_SIZE) + assert ex_mock.call_count == 0 + assert backtesting.get_pair_precision(pair, dt_utc(2017, 12, 15)) == (1e-8, TICK_SIZE) + assert ex_mock.call_count == 0 + + # Fallback to exchange logic + assert backtesting.get_pair_precision(pair, dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES) + assert ex_mock.call_count == 1 + assert backtesting.get_pair_precision("ETH/BTC", dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES) + assert ex_mock.call_count == 2 + + def test_backtest_abort(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf)