From 4849d5413ffff95854ac2881bad94ff7c1edb502 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:14:37 +0200 Subject: [PATCH 1/8] feat: add function to count "significant digits over time". --- freqtrade/data/btanalysis/__init__.py | 1 + .../data/btanalysis/historic_precision.py | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 freqtrade/data/btanalysis/historic_precision.py diff --git a/freqtrade/data/btanalysis/__init__.py b/freqtrade/data/btanalysis/__init__.py index 48fdcd9b5..520568558 100644 --- a/freqtrade/data/btanalysis/__init__.py +++ b/freqtrade/data/btanalysis/__init__.py @@ -25,6 +25,7 @@ from .bt_fileutils import ( trade_list_to_dataframe, update_backtest_metadata, ) +from .historic_precision import get_significant_digits_over_time from .trade_parallelism import ( analyze_trade_parallelism, evaluate_result_multi, diff --git a/freqtrade/data/btanalysis/historic_precision.py b/freqtrade/data/btanalysis/historic_precision.py new file mode 100644 index 000000000..96d12ba68 --- /dev/null +++ b/freqtrade/data/btanalysis/historic_precision.py @@ -0,0 +1,27 @@ +from pandas import DataFrame, Series + + +def get_significant_digits_over_time(candles: DataFrame) -> Series: + """ + Calculate the number of significant digits for candles over time. + It's using the Monthly maximum of the number of significant digits for each month. + :param candles: DataFrame with OHLCV data + :return: Series with the average number of significant digits for each month + """ + # count the number of significant digits for the open and close prices + for col in ["open", "high", "low", "close"]: + candles[f"{col}_count"] = ( + candles[col].round(14).astype(str).str.extract(r"\.(\d*[1-9])")[0].str.len() + ) + candles["max_count"] = candles[["open_count", "close_count", "high_count", "low_count"]].max( + axis=1 + ) + + candles1 = candles.set_index("date", drop=True) + # Group by month and calculate the average number of significant digits + monthly_count_avg1 = candles1["max_count"].resample("ME").max() + # monthly_open_count_avg + # convert monthly_open_count_avg from 5.0 to 0.00001, 4.0 to 0.0001, ... + monthly_open_count_avg = 1 / 10**monthly_count_avg1 + + return monthly_open_count_avg From b8f3f5e8d288188ae9af98f3d95e2995bbd291a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:28:10 +0200 Subject: [PATCH 2/8] fix: use month-start for the resample --- freqtrade/data/btanalysis/historic_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/btanalysis/historic_precision.py b/freqtrade/data/btanalysis/historic_precision.py index 96d12ba68..c4d186924 100644 --- a/freqtrade/data/btanalysis/historic_precision.py +++ b/freqtrade/data/btanalysis/historic_precision.py @@ -19,7 +19,7 @@ def get_significant_digits_over_time(candles: DataFrame) -> Series: candles1 = candles.set_index("date", drop=True) # Group by month and calculate the average number of significant digits - monthly_count_avg1 = candles1["max_count"].resample("ME").max() + monthly_count_avg1 = candles1["max_count"].resample("MS").max() # monthly_open_count_avg # convert monthly_open_count_avg from 5.0 to 0.00001, 4.0 to 0.0001, ... monthly_open_count_avg = 1 / 10**monthly_count_avg1 From 7d3fa41911a82c862bb84791385fdcafa2bd8ec3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:30:25 +0200 Subject: [PATCH 3/8] test: add tests for get_significant_digits_over_time --- tests/data/test_historic_precision.py | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/data/test_historic_precision.py diff --git a/tests/data/test_historic_precision.py b/tests/data/test_historic_precision.py new file mode 100644 index 000000000..a79be0f49 --- /dev/null +++ b/tests/data/test_historic_precision.py @@ -0,0 +1,92 @@ +# pragma pylint: disable=missing-docstring, C0103 + +from datetime import timezone + +import pandas as pd +from numpy import nan +from pandas import DataFrame, Timestamp + +from freqtrade.data.btanalysis.historic_precision import get_significant_digits_over_time + + +def test_get_significant_digits_over_time(): + """ + Test the get_significant_digits_over_time function with predefined data + """ + # Create test dataframe with different levels of precision + data = { + "date": [ + Timestamp("2020-01-01 00:00:00", tz=timezone.utc), + Timestamp("2020-01-02 00:00:00", tz=timezone.utc), + Timestamp("2020-01-03 00:00:00", tz=timezone.utc), + Timestamp("2020-01-15 00:00:00", tz=timezone.utc), + Timestamp("2020-01-16 00:00:00", tz=timezone.utc), + Timestamp("2020-01-31 00:00:00", tz=timezone.utc), + Timestamp("2020-02-01 00:00:00", tz=timezone.utc), + Timestamp("2020-02-15 00:00:00", tz=timezone.utc), + Timestamp("2020-03-15 00:00:00", tz=timezone.utc), + ], + "open": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34], + "high": [1.23457, 1.235, 1.24, 1.3, 1.23456, 1.235, 2.3457, 2.34, 2.34], + "low": [1.23455, 1.233, 1.22, 1.1, 1.23456, 1.233, 2.3455, 2.34, 2.34], + "close": [1.23456, 1.234, 1.23, 1.2, 1.23456, 1.234, 2.3456, 2.34, 2.34], + "volume": [100, 200, 300, 400, 500, 600, 700, 800, 900], + } + + candles = DataFrame(data) + + # Calculate significant digits + result = get_significant_digits_over_time(candles) + + # Check that the result is a pandas Series + assert isinstance(result, pd.Series) + + # Check that we have three months of data (Jan, Feb and March 2020 ) + assert len(result) == 3 + + # Before + assert result.asof("2019-01-01 00:00:00+00:00") is nan + # January should have 5 significant digits (based on 1.23456789 being the most precise value) + # which should be converted to 0.00001 + + assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001 + assert result.asof("2020-01-01 00:00:00+00:00") == 0.00001 + assert result.asof("2020-02-25 00:00:00+00:00") == 0.0001 + assert result.asof("2020-03-25 00:00:00+00:00") == 0.01 + assert result.asof("2020-04-01 00:00:00+00:00") == 0.01 + # Value far past the last date should be the last value + assert result.asof("2025-04-01 00:00:00+00:00") == 0.01 + + assert result.iloc[0] == 0.00001 + + +def test_get_significant_digits_over_time_real_data(testdatadir): + """ + Test the get_significant_digits_over_time function with real data from the testdatadir + """ + from freqtrade.data.history import load_pair_history + + # Load some test data from the testdata directory + pair = "UNITTEST/BTC" + timeframe = "1m" + + candles = load_pair_history( + datadir=testdatadir, + pair=pair, + timeframe=timeframe, + ) + + # Make sure we have test data + assert not candles.empty, "No test data found, cannot run test" + + # Calculate significant digits + result = get_significant_digits_over_time(candles) + + assert isinstance(result, pd.Series) + + # Verify that all values are between 0 and 1 (valid precision values) + assert all(result > 0) + assert all(result < 1) + + assert all(result <= 0.0001) + assert all(result >= 0.00000001) From 5a6b43da46c1fdd74d6c142c070c950d43c079c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:37:57 +0200 Subject: [PATCH 4/8] feat: use hsitoric price precision for improved accuracy closes #11203 --- freqtrade/optimize/backtesting.py | 45 +++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbc77604b..d1645e05b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,14 +9,18 @@ from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta -from numpy import nan -from pandas import DataFrame +from numpy import isnan, nan +from pandas import DataFrame, Series from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongShort from freqtrade.data import history -from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe +from freqtrade.data.btanalysis import ( + find_existing_backtest_stats, + get_significant_digits_over_time, + trade_list_to_dataframe, +) from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.data.metrics import combined_dataframes_with_rel_mean @@ -35,7 +39,7 @@ from freqtrade.exchange import ( price_to_precision, timeframe_to_seconds, ) -from freqtrade.exchange.exchange import Exchange +from freqtrade.exchange.exchange import TICK_SIZE, Exchange from freqtrade.ft_types import ( BacktestContentType, BacktestContentTypeIcomplete, @@ -121,6 +125,7 @@ class Backtesting: self.order_id_counter: int = 0 config["dry_run"] = True + self.price_pair_prec: dict[str, Series] = {} self.run_ids: dict[str, str] = {} self.strategylist: list[IStrategy] = [] self.all_bt_content: dict[str, BacktestContentType] = {} @@ -189,7 +194,6 @@ class Backtesting: self.fee = max(fee for fee in fees if fee is not None) logger.info(f"Using fee {self.fee:.4%} - worst case fee from exchange (lowest tier).") self.precision_mode = self.exchange.precisionMode - self.precision_mode_price = self.exchange.precision_mode_price if self.config.get("freqai_backtest_live_models", False): from freqtrade.freqai.utils import get_timerange_backtest_live_models @@ -316,6 +320,9 @@ class Backtesting: self.progress.set_new_value(1) self._load_bt_data_detail() + self.price_pair_prec = {} + for pair in self.pairlists.whitelist: + self.price_pair_prec[pair] = get_significant_digits_over_time(data[pair]) return data, self.timerange def _load_bt_data_detail(self) -> None: @@ -385,6 +392,22 @@ class Backtesting: else: self.futures_data = {} + def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float, int]: + """ + Get pair precision at that moment in time + :param pair: Pair to get precision for + :param current_time: Time to get precision for + :return: tuple of price precision, precision_mode_price for the pair at that given time. + """ + precision_series = self.price_pair_prec.get(pair) + if precision_series is not None: + precision = precision_series.asof(current_time) + + if not isnan(precision): + # Force tick size if we define the precision + return precision, TICK_SIZE + return self.exchange.get_precision_price(pair), self.exchange.precision_mode_price + def disable_database_use(self): disable_database_use(self.timeframe) @@ -793,7 +816,7 @@ class Backtesting: ) if rate is not None and rate != close_rate: close_rate = price_to_precision( - rate, trade.price_precision, self.precision_mode_price + rate, trade.price_precision, trade.precision_mode_price ) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately @@ -926,6 +949,7 @@ class Backtesting: trade: LocalTrade | None, order_type: str, price_precision: float | None, + precision_mode_price: int, ) -> tuple[float, float, float, float]: if order_type == "limit": new_rate = strategy_safe_wrapper( @@ -941,9 +965,7 @@ class Backtesting: # We can't place orders higher than current high (otherwise it'd be a stop limit entry) # which freqtrade does not support in live. if new_rate is not None and new_rate != propose_rate: - propose_rate = price_to_precision( - new_rate, price_precision, self.precision_mode_price - ) + propose_rate = price_to_precision(new_rate, price_precision, precision_mode_price) if direction == "short": propose_rate = max(propose_rate, row[LOW_IDX]) else: @@ -1036,7 +1058,7 @@ class Backtesting: pos_adjust = trade is not None and requested_rate is None stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0) - precision_price = self.exchange.get_precision_price(pair) + precision_price, precision_mode_price = self.get_pair_precision(pair, current_time) propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake( pair, @@ -1049,6 +1071,7 @@ class Backtesting: trade, order_type, precision_price, + precision_mode_price, ) # replace proposed rate if another rate was requested @@ -1124,7 +1147,7 @@ class Backtesting: amount_precision=precision_amount, price_precision=precision_price, precision_mode=self.precision_mode, - precision_mode_price=self.precision_mode_price, + precision_mode_price=precision_mode_price, contract_size=contract_size, orders=[], ) From ca67d3fb2cf4211d3168092a57b182c58b3c828d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:44:14 +0200 Subject: [PATCH 5/8] chore: improve price_precision logic resiliancy --- freqtrade/optimize/backtesting.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d1645e05b..3fc533e29 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -322,7 +322,8 @@ class Backtesting: self._load_bt_data_detail() self.price_pair_prec = {} for pair in self.pairlists.whitelist: - self.price_pair_prec[pair] = get_significant_digits_over_time(data[pair]) + if pair in data: + self.price_pair_prec[pair] = get_significant_digits_over_time(data[pair]) return data, self.timerange def _load_bt_data_detail(self) -> None: @@ -403,9 +404,9 @@ class Backtesting: if precision_series is not None: precision = precision_series.asof(current_time) - if not isnan(precision): - # Force tick size if we define the precision - return precision, TICK_SIZE + if not isnan(precision): + # Force tick size if we define the precision + return precision, TICK_SIZE return self.exchange.get_precision_price(pair), self.exchange.precision_mode_price def disable_database_use(self): From 081fa2209a82b1317722f85a067182af6eeec554 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 May 2025 20:45:52 +0200 Subject: [PATCH 6/8] refactor: correct naming of new function --- freqtrade/data/btanalysis/__init__.py | 2 +- freqtrade/data/btanalysis/historic_precision.py | 2 +- freqtrade/optimize/backtesting.py | 5 +++-- tests/data/test_historic_precision.py | 14 +++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/btanalysis/__init__.py b/freqtrade/data/btanalysis/__init__.py index 520568558..1889429f6 100644 --- a/freqtrade/data/btanalysis/__init__.py +++ b/freqtrade/data/btanalysis/__init__.py @@ -25,7 +25,7 @@ from .bt_fileutils import ( trade_list_to_dataframe, update_backtest_metadata, ) -from .historic_precision import get_significant_digits_over_time +from .historic_precision import get_tick_size_over_time from .trade_parallelism import ( analyze_trade_parallelism, evaluate_result_multi, diff --git a/freqtrade/data/btanalysis/historic_precision.py b/freqtrade/data/btanalysis/historic_precision.py index c4d186924..c8aa4fbee 100644 --- a/freqtrade/data/btanalysis/historic_precision.py +++ b/freqtrade/data/btanalysis/historic_precision.py @@ -1,7 +1,7 @@ from pandas import DataFrame, Series -def get_significant_digits_over_time(candles: DataFrame) -> Series: +def get_tick_size_over_time(candles: DataFrame) -> Series: """ Calculate the number of significant digits for candles over time. It's using the Monthly maximum of the number of significant digits for each month. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3fc533e29..6229a9c96 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,7 +18,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, IntOrInf, LongSho from freqtrade.data import history from freqtrade.data.btanalysis import ( find_existing_backtest_stats, - get_significant_digits_over_time, + get_tick_size_over_time, trade_list_to_dataframe, ) from freqtrade.data.converter import trim_dataframe, trim_dataframes @@ -323,7 +323,8 @@ class Backtesting: self.price_pair_prec = {} for pair in self.pairlists.whitelist: if pair in data: - self.price_pair_prec[pair] = get_significant_digits_over_time(data[pair]) + # Load price precision logic + self.price_pair_prec[pair] = get_tick_size_over_time(data[pair]) return data, self.timerange def _load_bt_data_detail(self) -> None: diff --git a/tests/data/test_historic_precision.py b/tests/data/test_historic_precision.py index a79be0f49..472218b8b 100644 --- a/tests/data/test_historic_precision.py +++ b/tests/data/test_historic_precision.py @@ -6,12 +6,12 @@ import pandas as pd from numpy import nan from pandas import DataFrame, Timestamp -from freqtrade.data.btanalysis.historic_precision import get_significant_digits_over_time +from freqtrade.data.btanalysis.historic_precision import get_tick_size_over_time -def test_get_significant_digits_over_time(): +def test_get_tick_size_over_time(): """ - Test the get_significant_digits_over_time function with predefined data + Test the get_tick_size_over_time function with predefined data """ # Create test dataframe with different levels of precision data = { @@ -36,7 +36,7 @@ def test_get_significant_digits_over_time(): candles = DataFrame(data) # Calculate significant digits - result = get_significant_digits_over_time(candles) + result = get_tick_size_over_time(candles) # Check that the result is a pandas Series assert isinstance(result, pd.Series) @@ -60,9 +60,9 @@ def test_get_significant_digits_over_time(): assert result.iloc[0] == 0.00001 -def test_get_significant_digits_over_time_real_data(testdatadir): +def test_get_tick_size_over_time_real_data(testdatadir): """ - Test the get_significant_digits_over_time function with real data from the testdatadir + Test the get_tick_size_over_time function with real data from the testdatadir """ from freqtrade.data.history import load_pair_history @@ -80,7 +80,7 @@ def test_get_significant_digits_over_time_real_data(testdatadir): assert not candles.empty, "No test data found, cannot run test" # Calculate significant digits - result = get_significant_digits_over_time(candles) + result = get_tick_size_over_time(candles) assert isinstance(result, pd.Series) From 53ba3ced06279c65318a7c31d917eae2fdd441f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 May 2025 11:47:24 +0200 Subject: [PATCH 7/8] test: add test for backtest "get price precision" logic --- tests/optimize/test_backtesting.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2973a4a49..9ea892ab8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -22,6 +22,7 @@ from freqtrade.data.history import get_timerange from freqtrade.enums import CandleType, ExitType, RunMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_next_date, timeframe_to_prev_date +from freqtrade.exchange.exchange_utils import DECIMAL_PLACES, TICK_SIZE from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename, get_strategy_run_id from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade, Trade @@ -348,6 +349,29 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: assert processed["UNITTEST/BTC"].equals(processed2["UNITTEST/BTC"]) +def test_get_pair_precision_bt(default_conf, mocker) -> None: + patch_exchange(mocker) + default_conf["timeframe"] = "30m" + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = "UNITTEST/BTC" + backtesting.pairlists._whitelist = [pair] + ex_mock = mocker.patch(f"{EXMS}.get_precision_price", return_value=1e-5) + data, timerange = backtesting.load_bt_data() + assert data + + assert backtesting.get_pair_precision(pair, dt_utc(2018, 1, 1)) == (1e-8, TICK_SIZE) + assert ex_mock.call_count == 0 + assert backtesting.get_pair_precision(pair, dt_utc(2017, 12, 15)) == (1e-8, TICK_SIZE) + assert ex_mock.call_count == 0 + + # Fallback to exchange logic + assert backtesting.get_pair_precision(pair, dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES) + assert ex_mock.call_count == 1 + assert backtesting.get_pair_precision("ETH/BTC", dt_utc(2017, 1, 15)) == (1e-5, DECIMAL_PLACES) + assert ex_mock.call_count == 2 + + def test_backtest_abort(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) From 2142b2aea35a82bfad8c2dfb441290335e7cb41f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 May 2025 11:49:03 +0200 Subject: [PATCH 8/8] chore: fix type-check --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6229a9c96..8d0073593 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -394,7 +394,7 @@ class Backtesting: else: self.futures_data = {} - def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float, int]: + def get_pair_precision(self, pair: str, current_time: datetime) -> tuple[float | None, int]: """ Get pair precision at that moment in time :param pair: Pair to get precision for