From 7ddaa09a2380dc2df81d0a570379a2808afa655f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 08:23:55 +0100 Subject: [PATCH 01/59] Refactor VolatilityFilter --- freqtrade/plugins/pairlist/VolatilityFilter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index ef72486e1..f18af2c97 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -105,13 +105,13 @@ class VolatilityFilter(IPairList): since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days)) candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) - if self._enabled: - for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._def_candletype)] if ( - p, '1d', self._def_candletype) in candles else None - if not self._validate_pair_loc(p, daily_candles): - pairlist.remove(p) - return pairlist + resulting_pairlist: List[str] = [] + for p in pairlist: + daily_candles = candles[(p, '1d', self._def_candletype)] if ( + p, '1d', self._def_candletype) in candles else None + if self._validate_pair_loc(p, daily_candles): + resulting_pairlist.append(p) + return resulting_pairlist def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: """ From 0bf73cc64b66cee16f9d0de95e10fdc553e989c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:11:43 +0100 Subject: [PATCH 02/59] Voliatilityfilter - sorting --- .../plugins/pairlist/VolatilityFilter.py | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index f18af2c97..70b48f7d2 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -37,6 +37,7 @@ class VolatilityFilter(IPairList): self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) self._refresh_period = pairlistconfig.get('refresh_period', 1440) self._def_candletype = self._config['candle_type_def'] + self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None) self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -89,6 +90,13 @@ class VolatilityFilter(IPairList): "description": "Maximum Volatility", "help": "Maximum volatility a pair must have to be considered.", }, + "sort_direction": { + "type": "option", + "default": None, + "options": [None, "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist", + }, **IPairList.refresh_period_parameter() } @@ -106,14 +114,34 @@ class VolatilityFilter(IPairList): candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) resulting_pairlist: List[str] = [] + volatilitys: Dict[str, float] = {} for p in pairlist: daily_candles = candles[(p, '1d', self._def_candletype)] if ( p, '1d', self._def_candletype) in candles else None - if self._validate_pair_loc(p, daily_candles): - resulting_pairlist.append(p) + + if daily_candles is not None and not daily_candles.empty: + volatility_avg = self._calculate_volatility(deepcopy(daily_candles)) + + if self._validate_pair_loc(p, volatility_avg): + resulting_pairlist.append(p) + if self._sort_direction: + volatilitys[p] = volatility_avg if not np.isnan(volatility_avg) else 0 + + if self._sort_direction: + resulting_pairlist = sorted(resulting_pairlist, + key=lambda p: volatilitys[p], + reverse=self._sort_direction == 'desc') return resulting_pairlist - def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + def _calculate_volatility(self, daily_candles: DataFrame) -> float: + returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) + returns.fillna(0, inplace=True) + + volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) + volatility_avg = volatility_series.mean() + return volatility_avg + + def _validate_pair_loc(self, pair: str, volatility_avg: float) -> bool: """ Validate trading range :param pair: Pair that's currently validated @@ -125,23 +153,17 @@ class VolatilityFilter(IPairList): return cached_res result = False - if daily_candles is not None and not daily_candles.empty: - returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) - returns.fillna(0, inplace=True) - volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) - volatility_avg = volatility_series.mean() - - if self._min_volatility <= volatility_avg <= self._max_volatility: - result = True - else: - self.log_once(f"Removed {pair} from whitelist, because volatility " - f"over {self._days} {plural(self._days, 'day')} " - f"is: {volatility_avg:.3f} " - f"which is not in the configured range of " - f"{self._min_volatility}-{self._max_volatility}.", - logger.info) - result = False - self._pair_cache[pair] = result + if self._min_volatility <= volatility_avg <= self._max_volatility: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because volatility " + f"over {self._days} {plural(self._days, 'day')} " + f"is: {volatility_avg:.3f} " + f"which is not in the configured range of " + f"{self._min_volatility}-{self._max_volatility}.", + logger.info) + result = False + self._pair_cache[pair] = result return result From 38ca58c728a752bf3eaa9164f1a724ea07807a95 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:12:38 +0100 Subject: [PATCH 03/59] Add verification for volatilityfilter --- freqtrade/plugins/pairlist/VolatilityFilter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 70b48f7d2..ff1525d70 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -47,6 +47,9 @@ class VolatilityFilter(IPairList): if self._days > candle_limit: raise OperationalException("VolatilityFilter requires lookback_days to not " f"exceed exchange max request size ({candle_limit})") + if self._sort_direction not in [None, 'asc', 'desc']: + raise OperationalException("VolatilityFilter requires sort_direction to be " + "either None (undefined), 'asc' or 'desc'") @property def needstickers(self) -> bool: From eaf70428c161a790a7e72fdf0549a78591a186f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:20:59 +0100 Subject: [PATCH 04/59] Improve volatility tests --- tests/plugins/test_pairlist.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d125f8896..64203d9a6 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -748,6 +748,32 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +def test_VolatilityFilter_error(mocker, whitelist_conf) -> None: + volatility_filter = {"method": "VolatilityFilter", "lookback_days": -1} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + exchange_mock = MagicMock() + exchange_mock.ohlcv_candle_limit = MagicMock(return_value=1000) + + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires lookback_days to be >= 1*"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + volatility_filter = {"method": "VolatilityFilter", "lookback_days": 2000} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires lookback_days to not exceed exchange max"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + volatility_filter = {"method": "VolatilityFilter", "sort_direction": "Random"} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + with pytest.raises(OperationalException, + match=r"VolatilityFilter requires sort_direction to be either " + r"None .*'asc'.*'desc'"): + PairListManager(exchange_mock, whitelist_conf, MagicMock()) + + def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: whitelist_conf['pairlists'] = [ {"method": "StaticPairList"}, From 31e254313425892abb6a51ccd205185b85e39693 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:30:38 +0100 Subject: [PATCH 05/59] Enhance generate_test_data with parametrizable random seed --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9c81c050d..c1c35fc9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,8 +142,8 @@ def generate_trades_history(n_rows, start_date: Optional[datetime] = None, days= return df -def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): - np.random.seed(42) +def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42): + np.random.seed(random_seed) base = np.random.normal(20, 2, size=size) if timeframe == '1y': @@ -174,9 +174,9 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): return df -def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05'): +def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42): """ Generates data in the ohlcv format used by ccxt """ - df = generate_test_data(timeframe, size, start) + df = generate_test_data(timeframe, size, start, random_seed) df['date'] = df.loc[:, 'date'].view(np.int64) // 1000 // 1000 return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns))) From 91ba4f642425e239d77acb4ef4a9f211dabe992d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:31:26 +0100 Subject: [PATCH 06/59] Add test for volatilityFilter sorting --- tests/plugins/test_pairlist.py | 35 +++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 64203d9a6..f16f5dce9 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -19,7 +19,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from freqtrade.util.datetime_helpers import dt_now -from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange, +from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data, get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re, num_log_has) @@ -774,6 +774,39 @@ def test_VolatilityFilter_error(mocker, whitelist_conf) -> None: PairListManager(exchange_mock, whitelist_conf, MagicMock()) +@pytest.mark.parametrize('sort_direction', ['asc', 'desc']) +def test_VolatilityFilter_sort(mocker, whitelist_conf, time_machine, sort_direction) -> None: + volatility_filter = {"method": "VolatilityFilter", "sort_direction": sort_direction} + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] + + df1 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=42) + df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=1) + assert not df1.equals(df2) + time_machine.move_to('2022-01-15 00:00:00+00:00') + + ohlcv_data = { + ('ETH/BTC', '1d', CandleType.SPOT): df1, + ('TKN/BTC', '1d', CandleType.SPOT): df2, + + } + mocker.patch.multiple( + EXMS, + exchange_has=MagicMock(return_value=True), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + exchange = get_patched_exchange(mocker, whitelist_conf) + exchange.ohlcv_candle_limit = MagicMock(return_value=1000) + plm = PairListManager(exchange, whitelist_conf, MagicMock()) + + assert exchange.ohlcv_candle_limit.call_count == 1 + plm.refresh_pairlist() + assert exchange.ohlcv_candle_limit.call_count == 1 + assert plm.whitelist == ( + ['ETH/BTC', 'TKN/BTC'] if sort_direction == 'asc' else ['TKN/BTC', 'ETH/BTC'] + ) + + def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: whitelist_conf['pairlists'] = [ {"method": "StaticPairList"}, From 866ff55d840d747e51995379d677e81735a6fad2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:34:42 +0100 Subject: [PATCH 07/59] document sort_direction mode --- docs/includes/pairlists.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index d1dd2cda7..844e30ff9 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -460,7 +460,7 @@ Volatility is the degree of historical variation of a pairs over time, it is mea This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. -This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. +This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. In the below example: If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h. @@ -477,6 +477,9 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove ] ``` +Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist. +When sorting, caching will be applied at the candle level - ignoring `refresh_period` (the candle's won't change anyway). + ### Full example of Pairlist Handlers The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value. From 88a2995b4c66baa0790493a1c291113bfc70354f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:39:46 +0100 Subject: [PATCH 08/59] Fix wrong typehint --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index ff1525d70..d71d13a4d 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -148,7 +148,7 @@ class VolatilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param daily_candles: Downloaded daily candles + :param volatility_avg: Average volatility :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache From 7af46628f8386bb6061090d2d04739906b675a9d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:50:54 +0100 Subject: [PATCH 09/59] Simplify rangeStability Filter --- .../plugins/pairlist/rangestabilityfilter.py | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 49fba59b9..d66ea92ec 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -2,9 +2,8 @@ Rate of change pairlist filter """ import logging -from copy import deepcopy from datetime import timedelta -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from cachetools import TTLCache from pandas import DataFrame @@ -103,45 +102,55 @@ class RangeStabilityFilter(IPairList): since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days + 1)) candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) - if self._enabled: - for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._def_candletype)] if ( - p, '1d', self._def_candletype) in candles else None - if not self._validate_pair_loc(p, daily_candles): - pairlist.remove(p) - return pairlist + resulting_pairlist: List[str] = [] - def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: - """ - Validate trading range - :param pair: Pair that's currently validated - :param daily_candles: Downloaded daily candles - :return: True if the pair can stay, false if it should be removed - """ + for p in pairlist: + daily_candles = candles.get((p, '1d', self._def_candletype), None) + + pct_change = self._calculate_rate_of_change(p, daily_candles) + + if pct_change is not None and self._validate_pair_loc(p, pct_change): + resulting_pairlist.append(p) + else: + self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) + + return resulting_pairlist + + def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> float: # Check symbol in cache - if (cached_res := self._pair_cache.get(pair, None)) is not None: - return cached_res - - result = True + if (pct_change := self._pair_cache.get(pair, None)) is not None: + return pct_change if daily_candles is not None and not daily_candles.empty: + highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 - if pct_change < self._min_rate_of_change: - self.log_once(f"Removed {pair} from whitelist, because rate of change " - f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " - f"which is below the threshold of {self._min_rate_of_change}.", - logger.info) - result = False - if self._max_rate_of_change: - if pct_change > self._max_rate_of_change: - self.log_once( - f"Removed {pair} from whitelist, because rate of change " - f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " - f"which is above the threshold of {self._max_rate_of_change}.", - logger.info) - result = False - self._pair_cache[pair] = result + self._pair_cache[pair] = pct_change + return pct_change else: - self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info) + return None + + def _validate_pair_loc(self, pair: str, pct_change: float) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param pct_change: Rate of change + :return: True if the pair can stay, false if it should be removed + """ + + result = True + if pct_change < self._min_rate_of_change: + self.log_once(f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is below the threshold of {self._min_rate_of_change}.", + logger.info) + result = False + if self._max_rate_of_change: + if pct_change > self._max_rate_of_change: + self.log_once( + f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is above the threshold of {self._max_rate_of_change}.", + logger.info) + result = False return result From 3677953d90e6b93e98bb9f6294a5959428750a46 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 13:54:52 +0100 Subject: [PATCH 10/59] Properly cache volatility-average --- docs/includes/pairlists.md | 1 - .../plugins/pairlist/VolatilityFilter.py | 42 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 844e30ff9..51c38fcce 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -478,7 +478,6 @@ If the volatility over the last 10 days is not in the range of 0.05-0.50, remove ``` Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting mode for this pairlist. -When sorting, caching will be applied at the candle level - ignoring `refresh_period` (the candle's won't change anyway). ### Full example of Pairlist Handlers diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index d71d13a4d..36f24af4b 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -119,16 +119,14 @@ class VolatilityFilter(IPairList): resulting_pairlist: List[str] = [] volatilitys: Dict[str, float] = {} for p in pairlist: - daily_candles = candles[(p, '1d', self._def_candletype)] if ( - p, '1d', self._def_candletype) in candles else None + daily_candles = candles.get((p, '1d', self._def_candletype), None) - if daily_candles is not None and not daily_candles.empty: - volatility_avg = self._calculate_volatility(deepcopy(daily_candles)) + volatility_avg = self._calculate_volatility(p, daily_candles) - if self._validate_pair_loc(p, volatility_avg): - resulting_pairlist.append(p) - if self._sort_direction: - volatilitys[p] = volatility_avg if not np.isnan(volatility_avg) else 0 + if volatility_avg is not None and self._validate_pair_loc(p, volatility_avg): + resulting_pairlist.append(p) + if self._sort_direction: + volatilitys[p] = volatility_avg if not np.isnan(volatility_avg) else 0 if self._sort_direction: resulting_pairlist = sorted(resulting_pairlist, @@ -136,13 +134,22 @@ class VolatilityFilter(IPairList): reverse=self._sort_direction == 'desc') return resulting_pairlist - def _calculate_volatility(self, daily_candles: DataFrame) -> float: - returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) - returns.fillna(0, inplace=True) + def _calculate_volatility(self, pair: str, daily_candles: DataFrame) -> float: + # Check symbol in cache + if (volatility_avg := self._pair_cache.get(pair, None)) is not None: + return volatility_avg - volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) - volatility_avg = volatility_series.mean() - return volatility_avg + if daily_candles is not None and not daily_candles.empty: + returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) + returns.fillna(0, inplace=True) + + volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) + volatility_avg = volatility_series.mean() + self._pair_cache[pair] = volatility_avg + + return volatility_avg + else: + return None def _validate_pair_loc(self, pair: str, volatility_avg: float) -> bool: """ @@ -151,11 +158,6 @@ class VolatilityFilter(IPairList): :param volatility_avg: Average volatility :return: True if the pair can stay, false if it should be removed """ - # Check symbol in cache - if (cached_res := self._pair_cache.get(pair, None)) is not None: - return cached_res - - result = False if self._min_volatility <= volatility_avg <= self._max_volatility: result = True @@ -167,6 +169,4 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility}.", logger.info) result = False - self._pair_cache[pair] = result - return result From 81de29a1e3b9f28dc6c77bd4c8f28d1fe66cbc83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:00:50 +0100 Subject: [PATCH 11/59] Improve conditions for removal of pairs --- freqtrade/plugins/pairlist/VolatilityFilter.py | 8 ++++++-- freqtrade/plugins/pairlist/rangestabilityfilter.py | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 36f24af4b..224dfcca8 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -123,8 +123,12 @@ class VolatilityFilter(IPairList): volatility_avg = self._calculate_volatility(p, daily_candles) - if volatility_avg is not None and self._validate_pair_loc(p, volatility_avg): - resulting_pairlist.append(p) + if volatility_avg is not None: + if self._validate_pair_loc(p, volatility_avg): + resulting_pairlist.append(p) + else: + self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) + if self._sort_direction: volatilitys[p] = volatility_avg if not np.isnan(volatility_avg) else 0 diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index d66ea92ec..ff0ec80e4 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -109,8 +109,9 @@ class RangeStabilityFilter(IPairList): pct_change = self._calculate_rate_of_change(p, daily_candles) - if pct_change is not None and self._validate_pair_loc(p, pct_change): - resulting_pairlist.append(p) + if pct_change is not None: + if self._validate_pair_loc(p, pct_change): + resulting_pairlist.append(p) else: self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) From 6a313aa9e38210823e9372acc07d301cda138dfb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:03:26 +0100 Subject: [PATCH 12/59] Improve help wording --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 224dfcca8..301a92b7d 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -98,7 +98,7 @@ class VolatilityFilter(IPairList): "default": None, "options": [None, "asc", "desc"], "description": "Sort pairlist", - "help": "Sort Pairlist", + "help": "Sort Pairlist ascending or descending by volatility.", }, **IPairList.refresh_period_parameter() } From 9dd59672756fe2a9f85ac5d6dcc34222fa8b4e9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:03:50 +0100 Subject: [PATCH 13/59] Add sorting capabilities to rangeStabilityFilter --- .../plugins/pairlist/rangestabilityfilter.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index ff0ec80e4..0bd35997c 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -3,7 +3,7 @@ Rate of change pairlist filter """ import logging from datetime import timedelta -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from cachetools import TTLCache from pandas import DataFrame @@ -31,6 +31,7 @@ class RangeStabilityFilter(IPairList): self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') self._refresh_period = pairlistconfig.get('refresh_period', 86400) self._def_candletype = self._config['candle_type_def'] + self._sort_direction: Optional[str] = pairlistconfig.get('sort_direction', None) self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -40,7 +41,9 @@ class RangeStabilityFilter(IPairList): if self._days > candle_limit: raise OperationalException("RangeStabilityFilter requires lookback_days to not " f"exceed exchange max request size ({candle_limit})") - + if self._sort_direction not in [None, 'asc', 'desc']: + raise OperationalException("RangeStabilityFilter requires sort_direction to be " + "either None (undefined), 'asc' or 'desc'") @property def needstickers(self) -> bool: """ @@ -86,6 +89,13 @@ class RangeStabilityFilter(IPairList): "description": "Maximum Rate of Change", "help": "Maximum rate of change to filter pairs.", }, + "sort_direction": { + "type": "option", + "default": None, + "options": [None, "asc", "desc"], + "description": "Sort pairlist", + "help": "Sort Pairlist ascending or descending by rate of change.", + }, **IPairList.refresh_period_parameter() } @@ -103,6 +113,7 @@ class RangeStabilityFilter(IPairList): candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms=since_ms) resulting_pairlist: List[str] = [] + pct_changes: Dict[str, float] = {} for p in pairlist: daily_candles = candles.get((p, '1d', self._def_candletype), None) @@ -112,9 +123,14 @@ class RangeStabilityFilter(IPairList): if pct_change is not None: if self._validate_pair_loc(p, pct_change): resulting_pairlist.append(p) + pct_changes[p] = pct_change else: self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) + if self._sort_direction: + resulting_pairlist = sorted(resulting_pairlist, + key=lambda p: pct_changes[p], + reverse=self._sort_direction == 'desc') return resulting_pairlist def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> float: From 2704f6e758e9a0a926bd2599f329dc51834e2cd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:05:25 +0100 Subject: [PATCH 14/59] Improve test --- tests/plugins/test_pairlist.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index f16f5dce9..43a99df33 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -789,10 +789,11 @@ def test_VolatilityFilter_sort(mocker, whitelist_conf, time_machine, sort_direct ('TKN/BTC', '1d', CandleType.SPOT): df2, } + ohlcv_mock = MagicMock(return_value=ohlcv_data) mocker.patch.multiple( EXMS, exchange_has=MagicMock(return_value=True), - refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + refresh_latest_ohlcv=ohlcv_mock, ) exchange = get_patched_exchange(mocker, whitelist_conf) @@ -801,11 +802,16 @@ def test_VolatilityFilter_sort(mocker, whitelist_conf, time_machine, sort_direct assert exchange.ohlcv_candle_limit.call_count == 1 plm.refresh_pairlist() + assert ohlcv_mock.call_count == 1 assert exchange.ohlcv_candle_limit.call_count == 1 assert plm.whitelist == ( ['ETH/BTC', 'TKN/BTC'] if sort_direction == 'asc' else ['TKN/BTC', 'ETH/BTC'] ) + plm.refresh_pairlist() + assert exchange.ohlcv_candle_limit.call_count == 1 + assert ohlcv_mock.call_count == 1 + def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: whitelist_conf['pairlists'] = [ From b972ee78ec34bff99d1b111452201be45bfa3a45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:41:06 +0100 Subject: [PATCH 15/59] Enhance rangeStability test --- tests/plugins/test_pairlist.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 43a99df33..6cf331c86 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1160,6 +1160,13 @@ def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): match='RangeStabilityFilter requires lookback_days to be >= 1'): get_patched_freqtradebot(mocker, default_conf) + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'sort_direction': 'something'}] + + with pytest.raises(OperationalException, + match='RangeStabilityFilter requires sort_direction to be either None.*'): + get_patched_freqtradebot(mocker, default_conf) + @pytest.mark.parametrize('min_rate_of_change,max_rate_of_change,expected_length', [ (0.01, 0.99, 5), From e82d9e2f5568c1f2847fb2cbf7cfe5d8cf944973 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:45:15 +0100 Subject: [PATCH 16/59] Test volatilityfilter with more pairs --- tests/plugins/test_pairlist.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 6cf331c86..5d7497273 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -775,18 +775,29 @@ def test_VolatilityFilter_error(mocker, whitelist_conf) -> None: @pytest.mark.parametrize('sort_direction', ['asc', 'desc']) -def test_VolatilityFilter_sort(mocker, whitelist_conf, time_machine, sort_direction) -> None: - volatility_filter = {"method": "VolatilityFilter", "sort_direction": sort_direction} - whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, volatility_filter] +def test_VolatilityFilter_sort( + mocker, whitelist_conf, tickers, time_machine, sort_direction) -> None: + whitelist_conf['pairlists'] = [ + {'method': 'VolumePairList', 'number_assets': 10}, + {"method": "VolatilityFilter", "sort_direction": sort_direction}] df1 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=42) - df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=1) + df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=2) + df3 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=3) + df4 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=4) + df5 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=5) + df6 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=6) + assert not df1.equals(df2) time_machine.move_to('2022-01-15 00:00:00+00:00') ohlcv_data = { ('ETH/BTC', '1d', CandleType.SPOT): df1, ('TKN/BTC', '1d', CandleType.SPOT): df2, + ('LTC/BTC', '1d', CandleType.SPOT): df3, + ('XRP/BTC', '1d', CandleType.SPOT): df4, + ('HOT/BTC', '1d', CandleType.SPOT): df5, + ('BLK/BTC', '1d', CandleType.SPOT): df6, } ohlcv_mock = MagicMock(return_value=ohlcv_data) @@ -794,22 +805,25 @@ def test_VolatilityFilter_sort(mocker, whitelist_conf, time_machine, sort_direct EXMS, exchange_has=MagicMock(return_value=True), refresh_latest_ohlcv=ohlcv_mock, + get_tickers=tickers + ) exchange = get_patched_exchange(mocker, whitelist_conf) exchange.ohlcv_candle_limit = MagicMock(return_value=1000) plm = PairListManager(exchange, whitelist_conf, MagicMock()) - assert exchange.ohlcv_candle_limit.call_count == 1 + assert exchange.ohlcv_candle_limit.call_count == 2 plm.refresh_pairlist() assert ohlcv_mock.call_count == 1 - assert exchange.ohlcv_candle_limit.call_count == 1 + assert exchange.ohlcv_candle_limit.call_count == 2 assert plm.whitelist == ( - ['ETH/BTC', 'TKN/BTC'] if sort_direction == 'asc' else ['TKN/BTC', 'ETH/BTC'] + ['XRP/BTC', 'ETH/BTC', 'LTC/BTC', 'TKN/BTC'] if sort_direction == 'asc' + else ['TKN/BTC', 'LTC/BTC', 'ETH/BTC', 'XRP/BTC'] ) plm.refresh_pairlist() - assert exchange.ohlcv_candle_limit.call_count == 1 + assert exchange.ohlcv_candle_limit.call_count == 2 assert ohlcv_mock.call_count == 1 From 67152ad48a1fbf7a621acb779fe4b7aa2b200cac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 14:56:42 +0100 Subject: [PATCH 17/59] Improve and parametrize pairlist tests --- .../plugins/pairlist/VolatilityFilter.py | 1 - .../plugins/pairlist/rangestabilityfilter.py | 1 + tests/plugins/test_pairlist.py | 35 ++++++++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 301a92b7d..ca375fcda 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -3,7 +3,6 @@ Volatility pairlist filter """ import logging import sys -from copy import deepcopy from datetime import timedelta from typing import Any, Dict, List, Optional diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 0bd35997c..730bb3d78 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -44,6 +44,7 @@ class RangeStabilityFilter(IPairList): if self._sort_direction not in [None, 'asc', 'desc']: raise OperationalException("RangeStabilityFilter requires sort_direction to be " "either None (undefined), 'asc' or 'desc'") + @property def needstickers(self) -> bool: """ diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 5d7497273..57affc731 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -774,12 +774,34 @@ def test_VolatilityFilter_error(mocker, whitelist_conf) -> None: PairListManager(exchange_mock, whitelist_conf, MagicMock()) -@pytest.mark.parametrize('sort_direction', ['asc', 'desc']) -def test_VolatilityFilter_sort( - mocker, whitelist_conf, tickers, time_machine, sort_direction) -> None: +@pytest.mark.parametrize('pairlist,expected_pairlist', [ + ({"method": "VolatilityFilter", "sort_direction": "asc"}, + ['XRP/BTC', 'ETH/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc"}, + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC', 'XRP/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc", 'min_volatility': 0.4}, + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "asc", 'min_volatility': 0.4}, + ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "desc", 'max_volatility': 0.5}, + ['LTC/BTC', 'ETH/BTC', 'XRP/BTC']), + ({"method": "VolatilityFilter", "sort_direction": "asc", 'max_volatility': 0.5}, + ['XRP/BTC', 'ETH/BTC', 'LTC/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "asc"}, + ['ETH/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "desc"}, + ['TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'ETH/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "asc", 'min_rate_of_change': 0.4}, + ['XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + ({"method": "RangeStabilityFilter", "sort_direction": "desc", 'min_rate_of_change': 0.4}, + ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']), +]) +def test_VolatilityFilter_RangeStabilityFilter_sort( + mocker, whitelist_conf, tickers, time_machine, pairlist, expected_pairlist) -> None: whitelist_conf['pairlists'] = [ {'method': 'VolumePairList', 'number_assets': 10}, - {"method": "VolatilityFilter", "sort_direction": sort_direction}] + pairlist + ] df1 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=42) df2 = generate_test_data('1d', 10, '2022-01-05 00:00:00+00:00', random_seed=2) @@ -817,10 +839,7 @@ def test_VolatilityFilter_sort( plm.refresh_pairlist() assert ohlcv_mock.call_count == 1 assert exchange.ohlcv_candle_limit.call_count == 2 - assert plm.whitelist == ( - ['XRP/BTC', 'ETH/BTC', 'LTC/BTC', 'TKN/BTC'] if sort_direction == 'asc' - else ['TKN/BTC', 'LTC/BTC', 'ETH/BTC', 'XRP/BTC'] - ) + assert plm.whitelist == expected_pairlist plm.refresh_pairlist() assert exchange.ohlcv_candle_limit.call_count == 2 From 817ad6440280ed2fd417307766676c1a89ab8577 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 15:00:28 +0100 Subject: [PATCH 18/59] Add docs for rangeStability sorting --- docs/includes/pairlists.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 51c38fcce..960f2d210 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -450,6 +450,8 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from ] ``` +Adding `"sort_direction": "asc"` or `"sort_direction": "desc"` enables sorting for this pairlist. + !!! Tip This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. Additionally, it can also be used to automatically remove pairs with extreme high/low variance over a given amount of time. From e80ad309f1b7b36ac23e5a7d09cb04c21f8f6465 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Feb 2024 15:04:54 +0100 Subject: [PATCH 19/59] Improve type safety, refactor volatilityfilter --- freqtrade/plugins/pairlist/VolatilityFilter.py | 10 +++++----- freqtrade/plugins/pairlist/rangestabilityfilter.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index ca375fcda..cdd171e91 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -95,7 +95,7 @@ class VolatilityFilter(IPairList): "sort_direction": { "type": "option", "default": None, - "options": [None, "asc", "desc"], + "options": ["", "asc", "desc"], "description": "Sort pairlist", "help": "Sort Pairlist ascending or descending by volatility.", }, @@ -125,19 +125,19 @@ class VolatilityFilter(IPairList): if volatility_avg is not None: if self._validate_pair_loc(p, volatility_avg): resulting_pairlist.append(p) + volatilitys[p] = ( + volatility_avg if volatility_avg and not np.isnan(volatility_avg) else 0 + ) else: self.log_once(f"Removed {p} from whitelist, no candles found.", logger.info) - if self._sort_direction: - volatilitys[p] = volatility_avg if not np.isnan(volatility_avg) else 0 - if self._sort_direction: resulting_pairlist = sorted(resulting_pairlist, key=lambda p: volatilitys[p], reverse=self._sort_direction == 'desc') return resulting_pairlist - def _calculate_volatility(self, pair: str, daily_candles: DataFrame) -> float: + def _calculate_volatility(self, pair: str, daily_candles: DataFrame) -> Optional[float]: # Check symbol in cache if (volatility_avg := self._pair_cache.get(pair, None)) is not None: return volatility_avg diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 730bb3d78..0480f60d0 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -93,7 +93,7 @@ class RangeStabilityFilter(IPairList): "sort_direction": { "type": "option", "default": None, - "options": [None, "asc", "desc"], + "options": ["", "asc", "desc"], "description": "Sort pairlist", "help": "Sort Pairlist ascending or descending by rate of change.", }, @@ -134,7 +134,7 @@ class RangeStabilityFilter(IPairList): reverse=self._sort_direction == 'desc') return resulting_pairlist - def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> float: + def _calculate_rate_of_change(self, pair: str, daily_candles: DataFrame) -> Optional[float]: # Check symbol in cache if (pct_change := self._pair_cache.get(pair, None)) is not None: return pct_change From 5402d276d61efb102d215d6fc1a84b61cd1eca59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Feb 2024 18:03:34 +0100 Subject: [PATCH 20/59] Add header to warning box, reorder Boxes --- docs/strategy-callbacks.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 0324e0915..2f04e906e 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -791,21 +791,21 @@ Returning a value more than the above (so remaining stake_amount would become ne If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that. Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order. -!!! Warning +!!! Warning "Stoploss calculation" Stoploss is still calculated from the initial opening price, not averaged price. Regular stoploss rules still apply (cannot move down). While `/stopentry` command stops the bot from entering new trades, the position adjustment feature will continue buying new orders on existing trades. -!!! Danger "Performance with many position adjustments" - Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively. - Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage. - Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance. - !!! Warning "Backtesting" During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected. This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle. +!!! Warning "Performance with many position adjustments" + Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively. + Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage. + Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance. + ``` python from freqtrade.persistence import Trade From 01266ed7eb90313c6f7db6f5c77405c300e2597c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Feb 2024 19:02:11 +0100 Subject: [PATCH 21/59] Align test results ... --- tests/optimize/test_backtesting_adjust_position.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 7f7bbb29f..ce2b73d02 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -76,7 +76,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], - 'close_timestamp': [1517265300000, 1517285400000], + 'close_timestamp': [1517265200000, 1517285400000], }) pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) data_pair = processed[pair] From d6b01a6ffe476797084ac82a1366a5147b2f9975 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 26 Feb 2024 19:17:50 +0100 Subject: [PATCH 22/59] Assert for exact equality --- .../optimize/test_backtesting_adjust_position.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index ce2b73d02..2a158acf3 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -57,28 +57,30 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> ), 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0), dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), - 'open_rate': [0.10401764894444211, 0.10302485], - 'close_rate': [0.10453904066847439, 0.103541], + 'open_rate': [0.10401764891917063, 0.10302485], + 'close_rate': [0.10453904064307624, 0.10354126528822055], 'fee_open': [0.0025, 0.0025], 'fee_close': [0.0025, 0.0025], 'trade_duration': [200, 40], 'profit_ratio': [0.0, 0.0], 'profit_abs': [0.0, 0.0], 'exit_reason': [ExitType.ROI.value, ExitType.ROI.value], - 'initial_stop_loss_abs': [0.0940005, 0.09272236], + 'initial_stop_loss_abs': [0.0940005, 0.092722365], 'initial_stop_loss_ratio': [-0.1, -0.1], - 'stop_loss_abs': [0.0940005, 0.09272236], + 'stop_loss_abs': [0.0940005, 0.092722365], 'stop_loss_ratio': [-0.1, -0.1], 'min_rate': [0.10370188, 0.10300000000000001], - 'max_rate': [0.10481985, 0.1038888], + 'max_rate': [0.10481985, 0.10388887000000001], 'is_open': [False, False], 'enter_tag': ['', ''], 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], - 'close_timestamp': [1517265200000, 1517285400000], + 'close_timestamp': [1517263200000, 1517285400000], }) - pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) + results_no = results.drop(columns=['orders']) + pd.testing.assert_frame_equal(results_no, expected, check_exact=True) + data_pair = processed[pair] assert len(results.iloc[0]['orders']) == 6 assert len(results.iloc[1]['orders']) == 2 From bd7edfba977d5749d3ac3edab3c0a787b072b41d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:27:08 +0000 Subject: [PATCH 23/59] Bump pandas from 2.1.4 to 2.2.1 Bumps [pandas](https://github.com/pandas-dev/pandas) from 2.1.4 to 2.2.1. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Commits](https://github.com/pandas-dev/pandas/compare/v2.1.4...v2.2.1) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 94f63d033..0477751cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.26.4 -pandas==2.1.4 +pandas==2.2.1 pandas-ta==0.3.14b ccxt==4.2.51 From b1015172c7b5f690b821bdf7010f45ecaa827172 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 05:55:26 +0100 Subject: [PATCH 24/59] Update test for fixed pandas behavior --- tests/strategy/test_interface.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 790f5d255..645cae887 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1022,22 +1022,22 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): @pytest.mark.parametrize('function,raises', [ - ('populate_entry_trend', True), + ('populate_entry_trend', False), ('advise_entry', False), - ('populate_exit_trend', True), + ('populate_exit_trend', False), ('advise_exit', False), ]) -def test_pandas_warning_direct(ohlcv_history, function, raises): +def test_pandas_warning_direct(ohlcv_history, function, raises, recwarn): df = _STRATEGY.populate_indicators(ohlcv_history, {'pair': 'ETH/BTC'}) if raises: - with pytest.warns(FutureWarning): - # Test for Future warning - # FutureWarning: Setting an item of incompatible dtype is - # deprecated and will raise in a future error of pandas - # https://github.com/pandas-dev/pandas/issues/56503 - getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + assert len(recwarn) == 1 + # https://github.com/pandas-dev/pandas/issues/56503 + # Fixed in 2.2.x + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) else: + assert len(recwarn) == 0 + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) From 5912d87b658995665c5c2eb12e61d329e0adf44d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 06:05:01 +0100 Subject: [PATCH 25/59] Pandas - update view to astype --- freqtrade/data/history/jsondatahandler.py | 2 +- freqtrade/rpc/rpc.py | 2 +- tests/conftest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 9a02a7769..baa0c10a5 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -37,7 +37,7 @@ class JsonDataHandler(IDataHandler): self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int - _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 + _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000 # Reset index, select only appropriate columns and save as json _data.reset_index(drop=True).loc[:, self._columns].to_json( diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2317ee1a9..6e8447d29 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1155,7 +1155,7 @@ class RPC: } if has_content: - dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000 + dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 # Move signal close to separate column when signal for easy plotting for sig_type in signals.keys(): if sig_type in dataframe.columns: diff --git a/tests/conftest.py b/tests/conftest.py index 9c81c050d..a322bfd77 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -177,7 +177,7 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05'): """ Generates data in the ohlcv format used by ccxt """ df = generate_test_data(timeframe, size, start) - df['date'] = df.loc[:, 'date'].view(np.int64) // 1000 // 1000 + df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000 return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns))) From ce2f4f89c4bbc7b6b7ae028af477dfa28862d424 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 06:07:21 +0100 Subject: [PATCH 26/59] update pandas deprecation to_hdf --- freqtrade/data/history/hdf5datahandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index d22fd9e31..b118bd7e0 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -35,7 +35,7 @@ class HDF5DataHandler(IDataHandler): self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( - filename, key, mode='a', complevel=9, complib='blosc', + filename, key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['date'] ) @@ -110,7 +110,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_trades_key(pair) data.to_hdf( - self._pair_trades_filename(self._datadir, pair), key, + self._pair_trades_filename(self._datadir, pair), key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) From 0021e2c2052a6643d53d59bfaa1bc9df88a9ee12 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 06:08:25 +0100 Subject: [PATCH 27/59] fillna needs explicit type. --- freqtrade/optimize/analysis/lookahead_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index 1d2b9db48..00f83a46b 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -107,9 +107,9 @@ class LookaheadAnalysisSubFunctions: csv_df = add_or_update_row(csv_df, new_row_data) # Fill NaN values with a default value (e.g., 0) - csv_df['total_signals'] = csv_df['total_signals'].fillna(0) - csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0) - csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0) + csv_df['total_signals'] = csv_df['total_signals'].astype(int).fillna(0) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int).fillna(0) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int).fillna(0) # Convert columns to integers csv_df['total_signals'] = csv_df['total_signals'].astype(int) From c0e9726f493adde27b42c60d7965db78019c4161 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 06:08:48 +0100 Subject: [PATCH 28/59] don't use "1M" - but be explicit in the intend --- freqtrade/optimize/optimize_reports/optimize_reports.py | 2 +- tests/optimize/test_optimize_reports.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 47a13dcd8..47aab2a62 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -215,7 +215,7 @@ def _get_resample_from_period(period: str) -> str: # Weekly defaulting to Monday. return '1W-MON' if period == 'month': - return '1M' + return '1ME' raise ValueError(f"Period {period} is not supported.") diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 0f190f3f5..e3603849d 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -498,7 +498,7 @@ def test__get_resample_from_period(): assert _get_resample_from_period('day') == '1d' assert _get_resample_from_period('week') == '1W-MON' - assert _get_resample_from_period('month') == '1M' + assert _get_resample_from_period('month') == '1ME' with pytest.raises(ValueError, match=r"Period noooo is not supported."): _get_resample_from_period('noooo') From 883f27d99e4150ada229b3e8b41a20087442ba75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Feb 2024 07:01:16 +0100 Subject: [PATCH 29/59] Version bump to 2024.3-dev --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 7c699d643..fa5d9214e 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2024.2-dev' +__version__ = '2024.3-dev' if 'dev' in __version__: from pathlib import Path From e988995d71a512e2485f3663c9a876c9c21855d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Feb 2024 07:22:40 +0100 Subject: [PATCH 30/59] Handle NaN funding fees closes #9831 --- freqtrade/exchange/exchange.py | 5 +++-- tests/exchange/test_exchange.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7f7fccca8..d1ac47df4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -8,7 +8,7 @@ import logging import signal from copy import deepcopy from datetime import datetime, timedelta, timezone -from math import floor +from math import floor, isnan from threading import Lock from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union @@ -2916,7 +2916,8 @@ class Exchange: if not df.empty: df1 = df[(df['date'] >= open_date) & (df['date'] <= close_date)] fees = sum(df1['open_fund'] * df1['open_mark'] * amount) - + if isnan(fees): + fees = 0.0 # Negate fees for longs as funding_fees expects it this way based on live endpoints. return fees if is_short else -fees diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 5c4879a32..168cf512d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock, PropertyMock, patch import ccxt import pytest +from numpy import NaN from pandas import DataFrame from freqtrade.enums import CandleType, MarginMode, RunMode, TradingMode @@ -4203,6 +4204,7 @@ def test_get_max_leverage_from_margin(default_conf, mocker, pair, nominal_value, (10, 0.0001, 2.0, 1.0, 0.002, 0.002), (10, 0.0002, 2.0, 0.01, 0.004, 0.00004), (10, 0.0002, 2.5, None, 0.005, None), + (10, 0.0002, NaN, None, 0.0, None), ]) def test_calculate_funding_fees( default_conf, From cdfeae9f904a1f615723ad3ba2136acc30061b52 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Feb 2024 07:31:22 +0100 Subject: [PATCH 31/59] Update merge to "left" to avoid creating non-existing mark candles --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d1ac47df4..d17b442ab 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2887,7 +2887,7 @@ class Exchange: else: # Fill up missing funding_rate candles with fallback value combined = mark_rates.merge( - funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"] + funding_rates, on='date', how="left", suffixes=["_mark", "_fund"] ) combined['open_fund'] = combined['open_fund'].fillna(futures_funding_rate) return combined diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 168cf512d..34d4ca4c6 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4314,8 +4314,8 @@ def test_combine_funding_and_mark( assert len(df) == 1 # Empty funding rates - funding_rates = DataFrame([], columns=['date', 'open']) - df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate) + funding_rates2 = DataFrame([], columns=['date', 'open']) + df = exchange.combine_funding_and_mark(funding_rates2, mark_rates, futures_funding_rate) if futures_funding_rate is not None: assert len(df) == 3 assert df.iloc[0]['open_fund'] == futures_funding_rate @@ -4324,6 +4324,12 @@ def test_combine_funding_and_mark( else: assert len(df) == 0 + # Empty mark candles + mark_candles = DataFrame([], columns=['date', 'open']) + df = exchange.combine_funding_and_mark(funding_rates, mark_candles, futures_funding_rate) + + assert len(df) == 0 + @pytest.mark.parametrize('exchange,rate_start,rate_end,d1,d2,amount,expected_fees', [ ('binance', 0, 2, "2021-09-01 01:00:00", "2021-09-01 04:00:00", 30.0, 0.0), From 46e616f9975b99de2fe7fb203c58ff0111212e1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 19:32:18 +0100 Subject: [PATCH 32/59] Remove defaults for converter - they're always provided and necessary. --- freqtrade/commands/data_commands.py | 3 ++- freqtrade/data/converter/trade_converter.py | 8 ++++---- freqtrade/data/history/history_utils.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 33069885a..d3600e3ef 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,7 +8,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, convert_trades_to_ohlcv) from freqtrade.data.history import download_data_main -from freqtrade.enums import RunMode, TradingMode +from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver @@ -69,6 +69,7 @@ def start_convert_trades(args: Dict[str, Any]) -> None: datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT) ) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index bd4efb77e..117f65bc6 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -88,10 +88,10 @@ def convert_trades_to_ohlcv( timeframes: List[str], datadir: Path, timerange: TimeRange, - erase: bool = False, - data_format_ohlcv: str = 'feather', - data_format_trades: str = 'feather', - candle_type: CandleType = CandleType.SPOT + erase: bool, + data_format_ohlcv: str, + data_format_trades: str, + candle_type: CandleType, ) -> None: """ Convert stored trades data to ohlcv data diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index ff6c2561d..27e229973 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -529,6 +529,7 @@ def download_data_main(config: Config) -> None: datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT), ) else: if not exchange.get_option('ohlcv_has_history', True): From 5dee60921f41cc4efd562c6047ea8cc97a0ff8e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 19:42:33 +0100 Subject: [PATCH 33/59] Fix test for convert_trades_to_ohlcv --- tests/data/test_converter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 08fc785aa..2202ada44 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data @@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has(msg, caplog) From bdd63aa1d661be6e54500329a0f50fc614d2f3e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 20:17:43 +0100 Subject: [PATCH 34/59] FIx futures trades pair download directory --- freqtrade/data/history/idatahandler.py | 5 +++++ tests/commands/test_commands.py | 5 ----- tests/data/test_download_data.py | 4 ---- tests/data/test_history.py | 1 + 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 47c2dd838..01c244f38 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -266,6 +266,11 @@ class IDataHandler(ABC): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: pair_s = misc.pair_to_filename(pair) + if ':' in pair: + # Futures pair ... + # TODO: this should not rely on ";" in the pairname. + datadir = datadir.joinpath('futures') + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cdad46407..1ab9d2202 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -820,11 +820,6 @@ def test_download_data_trades(mocker): "--trading-mode", "futures", "--dl-trades" ] - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - pargs = get_args(args) - pargs['config'] = None - start_download_data(pargs) def test_download_data_data_invalid(mocker): diff --git a/tests/data/test_download_data.py b/tests/data/test_download_data.py index 97640d01c..1518b28f3 100644 --- a/tests/data/test_download_data.py +++ b/tests/data/test_download_data.py @@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker): "trading_mode": "futures", }) - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - download_data_main(config) - def test_download_data_main_data_invalid(mocker): patch_exchange(mocker, id="kraken") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index a48d34aee..106babd63 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -170,6 +170,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): @pytest.mark.parametrize("pair,expected_result", [ ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("ETH/USDT:USDT", 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), From 75c84bfe654497f9d64f193cfc26591c99118e76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 08:23:38 +0100 Subject: [PATCH 35/59] Only show a subset of list-exchanges output in the docs (it's potentially missleading, and changes all the time - so was probably outdated). --- docs/utils.md | 218 +++++++------------------------------------------- 1 file changed, 30 insertions(+), 188 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 202526afe..ea52737d6 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -219,207 +219,49 @@ optional arguments: -a, --all Print all exchanges known to the ccxt library. ``` -* Example: see exchanges available for the bot: +Example: see exchanges available for the bot: + ``` $ freqtrade list-exchanges Exchanges available for Freqtrade: -Exchange name Valid reason ---------------- ------- -------------------------------------------- -aax True -ascendex True missing opt: fetchMyTrades -bequant True -bibox True -bigone True -binance True -binanceus True -bitbank True missing opt: fetchTickers -bitcoincom True -bitfinex True -bitforex True missing opt: fetchMyTrades, fetchTickers -bitget True -bithumb True missing opt: fetchMyTrades -bitkk True missing opt: fetchMyTrades -bitmart True -bitmax True missing opt: fetchMyTrades -bitpanda True -bitvavo True -bitz True missing opt: fetchMyTrades -btcalpha True missing opt: fetchTicker, fetchTickers -btcmarkets True missing opt: fetchTickers -buda True missing opt: fetchMyTrades, fetchTickers -bw True missing opt: fetchMyTrades, fetchL2OrderBook -bybit True -bytetrade True -cdax True -cex True missing opt: fetchMyTrades -coinbaseprime True missing opt: fetchTickers -coinbasepro True missing opt: fetchTickers -coinex True -crex24 True -deribit True -digifinex True -equos True missing opt: fetchTicker, fetchTickers -eterbase True -fcoin True missing opt: fetchMyTrades, fetchTickers -fcoinjp True missing opt: fetchMyTrades, fetchTickers -gateio True -gemini True -gopax True -hbtc True -hitbtc True -huobijp True -huobipro True -idex True -kraken True -kucoin True -lbank True missing opt: fetchMyTrades -mercado True missing opt: fetchTickers -ndax True missing opt: fetchTickers -novadax True -okcoin True -okex True -probit True -qtrade True -stex True -timex True -upbit True missing opt: fetchMyTrades -vcc True -zb True missing opt: fetchMyTrades - +Exchange name Supported Markets Reason +------------------ ----------- ---------------------- ------------------------------------------------------------------------ +binance Official spot, isolated futures +bitmart Official spot +bybit spot, isolated futures +gate Official spot, isolated futures +htx Official spot +huobi spot +kraken Official spot +okx Official spot, isolated futures ``` +!!! info "" + Output reduced for clarity - supported and available exchanges may change over time. + !!! Note "missing opt exchanges" Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will). -* Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade): +Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade) + ``` $ freqtrade list-exchanges -a All exchanges supported by the ccxt library: -Exchange name Valid reason ------------------- ------- --------------------------------------------------------------------------------------- -aax True -aofex False missing: fetchOrder -ascendex True missing opt: fetchMyTrades -bequant True -bibox True -bigone True -binance True -binanceus True -bit2c False missing: fetchOrder, fetchOHLCV -bitbank True missing opt: fetchTickers -bitbay False missing: fetchOrder -bitcoincom True -bitfinex True -bitfinex2 False missing: fetchOrder -bitflyer False missing: fetchOrder, fetchOHLCV -bitforex True missing opt: fetchMyTrades, fetchTickers -bitget True -bithumb True missing opt: fetchMyTrades -bitkk True missing opt: fetchMyTrades -bitmart True -bitmax True missing opt: fetchMyTrades -bitmex False Various reasons. -bitpanda True -bitso False missing: fetchOHLCV -bitstamp True missing opt: fetchTickers -bitstamp1 False missing: fetchOrder, fetchOHLCV -bitvavo True -bitz True missing opt: fetchMyTrades -bl3p False missing: fetchOrder, fetchOHLCV -bleutrade False missing: fetchOrder -braziliex False missing: fetchOHLCV -btcalpha True missing opt: fetchTicker, fetchTickers -btcbox False missing: fetchOHLCV -btcmarkets True missing opt: fetchTickers -btctradeua False missing: fetchOrder, fetchOHLCV -btcturk False missing: fetchOrder -buda True missing opt: fetchMyTrades, fetchTickers -bw True missing opt: fetchMyTrades, fetchL2OrderBook -bybit True -bytetrade True -cdax True -cex True missing opt: fetchMyTrades -chilebit False missing: fetchOrder, fetchOHLCV -coinbase False missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV -coinbaseprime True missing opt: fetchTickers -coinbasepro True missing opt: fetchTickers -coincheck False missing: fetchOrder, fetchOHLCV -coinegg False missing: fetchOHLCV -coinex True -coinfalcon False missing: fetchOHLCV -coinfloor False missing: fetchOrder, fetchOHLCV -coingi False missing: fetchOrder, fetchOHLCV -coinmarketcap False missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV -coinmate False missing: fetchOHLCV -coinone False missing: fetchOHLCV -coinspot False missing: fetchOrder, cancelOrder, fetchOHLCV -crex24 True -currencycom False missing: fetchOrder -delta False missing: fetchOrder -deribit True -digifinex True -equos True missing opt: fetchTicker, fetchTickers -eterbase True -exmo False missing: fetchOrder -exx False missing: fetchOHLCV -fcoin True missing opt: fetchMyTrades, fetchTickers -fcoinjp True missing opt: fetchMyTrades, fetchTickers -flowbtc False missing: fetchOrder, fetchOHLCV -foxbit False missing: fetchOrder, fetchOHLCV -gateio True -gemini True -gopax True -hbtc True -hitbtc True -hollaex False missing: fetchOrder -huobijp True -huobipro True -idex True -independentreserve False missing: fetchOHLCV -indodax False missing: fetchOHLCV -itbit False missing: fetchOHLCV -kraken True -kucoin True -kuna False missing: fetchOHLCV -lakebtc False missing: fetchOrder, fetchOHLCV -latoken False missing: fetchOrder, fetchOHLCV -lbank True missing opt: fetchMyTrades -liquid False missing: fetchOHLCV -luno False missing: fetchOHLCV -lykke False missing: fetchOHLCV -mercado True missing opt: fetchTickers -mixcoins False missing: fetchOrder, fetchOHLCV -ndax True missing opt: fetchTickers -novadax True -oceanex False missing: fetchOHLCV -okcoin True -okex True -paymium False missing: fetchOrder, fetchOHLCV -phemex False Does not provide history. -poloniex False missing: fetchOrder -probit True -qtrade True -rightbtc False missing: fetchOrder -ripio False missing: fetchOHLCV -southxchange False missing: fetchOrder, fetchOHLCV -stex True -surbitcoin False missing: fetchOrder, fetchOHLCV -therock False missing: fetchOHLCV -tidebit False missing: fetchOrder -tidex False missing: fetchOHLCV -timex True -upbit True missing opt: fetchMyTrades -vbtc False missing: fetchOrder, fetchOHLCV -vcc True -wavesexchange False missing: fetchOrder -whitebit False missing: fetchOrder, cancelOrder, createOrder, fetchBalance -xbtce False missing: fetchOrder, fetchOHLCV -xena False missing: fetchOrder -yobit False missing: fetchOHLCV -zaif False missing: fetchOrder, fetchOHLCV -zb True missing opt: fetchMyTrades +Exchange name Valid Supported Markets Reason +------------------ ------- ----------- ---------------------- --------------------------------------------------------------------------------- +binance True Official spot, isolated futures +bitflyer False spot missing: fetchOrder. missing opt: fetchTickers. +bitmart True Official spot +bybit True spot, isolated futures +gate True Official spot, isolated futures +htx True Official spot +kraken True Official spot +okx True Official spot, isolated futures ``` +!!! info "" + Reduced output - supported and available exchanges may change over time. + ## List Timeframes Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange. From f8cc2a6e74ee5de6db16b3d234aea956df6daded Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 12:01:01 +0100 Subject: [PATCH 36/59] Fix typo in doc header --- docs/updating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/updating.md b/docs/updating.md index 1e5dc8ffe..5841d205a 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -6,7 +6,7 @@ To update your freqtrade installation, please use one of the below methods, corr Breaking changes / changed behavior will be documented in the changelog that is posted alongside every release. For the develop branch, please follow PR's to avoid being surprised by changes. -## docker +## Docker !!! Note "Legacy installations using the `master` image" We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable` From b6040e270fa253fc089230decf37d7cce98d61a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:08:58 +0100 Subject: [PATCH 37/59] Update data handlers to accept trading_mode for trade data related functions --- freqtrade/data/history/featherdatahandler.py | 14 ++++++---- freqtrade/data/history/hdf5datahandler.py | 14 ++++++---- freqtrade/data/history/idatahandler.py | 29 ++++++++++++++------ freqtrade/data/history/jsondatahandler.py | 14 ++++++---- freqtrade/data/history/parquetdatahandler.py | 16 +++++++---- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/freqtrade/data/history/featherdatahandler.py b/freqtrade/data/history/featherdatahandler.py index 44d337836..6d57dbed7 100644 --- a/freqtrade/data/history/featherdatahandler.py +++ b/freqtrade/data/history/featherdatahandler.py @@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4') @@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index b118bd7e0..cb2cdd884 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -6,7 +6,7 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: pd.DataFrame) -> None: + def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ key = self._pair_trades_key(pair) data.to_hdf( - self._pair_trades_filename(self._datadir, pair), key=key, + self._pair_trades_filename(self._datadir, pair, trading_mode), key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) @@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> pd.DataFrame: """ Load a pair from h5 file. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ key = self._pair_trades_key(pair) - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 01c244f38..bcb31a7c8 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -172,12 +172,13 @@ class IDataHandler(ABC): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ @abstractmethod @@ -190,45 +191,55 @@ class IDataHandler(ABC): """ @abstractmethod - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - def trades_store(self, pair: str, data: DataFrame) -> None: + def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ # Filter on expected columns (will remove the actual date column). - self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS]) + self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode) - def trades_purge(self, pair: str) -> bool: + def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. + :param trading_mode: Trading mode to use (used to determine the filename) :return: True when deleted, false if file did not exist. """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if filename.exists(): filename.unlink() return True return False - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json Removes duplicates in the process. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange)) + trades = trades_df_remove_duplicates( + self._trades_load(pair, trading_mode, timerange=timerange) + ) trades = trades_convert_types(trades) return trades @@ -264,7 +275,7 @@ class IDataHandler(ABC): return filename @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path: pair_s = misc.pair_to_filename(pair) if ':' in pair: # Futures pair ... diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index baa0c10a5..2d0333fed 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,7 +8,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) trades = data.values.tolist() misc.file_dump_json(filename, trades, is_zip=self._use_zip) @@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) tradesdata = misc.file_load_json(filename) if not tradesdata: diff --git a/freqtrade/data/history/parquetdatahandler.py b/freqtrade/data/history/parquetdatahandler.py index c0b0cad63..01becdc84 100644 --- a/freqtrade/data/history/parquetdatahandler.py +++ b/freqtrade/data/history/parquetdatahandler.py @@ -4,8 +4,8 @@ from typing import Optional from pandas import DataFrame, read_parquet, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList -from freqtrade.enums import CandleType +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_parquet(filename) @@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) From 43103f51e55aada44c81a8a23fd29d886a3e3244 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:10:57 +0100 Subject: [PATCH 38/59] Update functions that use datahandler trade methods --- freqtrade/data/converter/trade_converter.py | 15 ++++++------ .../data/converter/trade_converter_kraken.py | 3 ++- freqtrade/data/history/history_utils.py | 24 ++++++++++++------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 117f65bc6..0e5050a05 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, Config, TradeList) -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException @@ -104,9 +104,9 @@ def convert_trades_to_ohlcv( logger.info(f"About to convert pairs: '{', '.join(pairs)}', " f"intervals: '{', '.join(timeframes)}' to {datadir}") - + trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT for pair in pairs: - trades = data_handler_trades.trades_load(pair) + trades = data_handler_trades.trades_load(pair, trading_mode) for timeframe in timeframes: if erase: if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): @@ -144,11 +144,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er if 'pairs' not in config: config['pairs'] = src.trades_get_pairs(config['datadir']) logger.info(f"Converting trades for {config['pairs']}") - + trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) for pair in config['pairs']: - data = src.trades_load(pair=pair) + data = src.trades_load(pair, trading_mode) logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) + trg.trades_store(pair, data, trading_mode) + if erase and convert_from != convert_to: logger.info(f"Deleting source Trade data for {pair}.") - src.trades_purge(pair=pair) + src.trades_purge(pair, trading_mode) diff --git a/freqtrade/data/converter/trade_converter_kraken.py b/freqtrade/data/converter/trade_converter_kraken.py index b0fa11c25..80bd917af 100644 --- a/freqtrade/data/converter/trade_converter_kraken.py +++ b/freqtrade/data/converter/trade_converter_kraken.py @@ -7,6 +7,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C from freqtrade.data.converter.trade_converter import (trades_convert_types, trades_df_remove_duplicates) from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver @@ -79,4 +80,4 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str): f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to " f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}") - data_handler.trades_store(pair, trades_df) + data_handler.trades_store(pair, trades_df, TradingMode.SPOT) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 27e229973..3f9468f7a 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o ohlcv_to_dataframe, trades_df_remove_duplicates, trades_list_to_df) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange, pair: str, *, new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, - data_handler: IDataHandler + data_handler: IDataHandler, + trading_mode: TradingMode, ) -> bool: """ Download trade history from the exchange. @@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 - trades = data_handler.trades_load(pair) + trades = data_handler.trades_load(pair, trading_mode) # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS # DEFAULT_TRADES_COLUMNS: 0 -> timestamp @@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange, trades = concat([trades, new_trades_df], axis=0) # Remove duplicates to make sure we're not storing data we don't need trades = trades_df_remove_duplicates(trades) - data_handler.trades_store(pair, data=trades) + data_handler.trades_store(pair, trades, trading_mode) logger.debug("New Start: %s", 'None' if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}") @@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, new_pairs_days: int = 30, - erase: bool = False, data_format: str = 'feather') -> List[str]: + timerange: TimeRange, trading_mode: TradingMode, + new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'feather', + ) -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: continue if erase: - if data_handler.trades_purge(pair): + if data_handler.trades_purge(pair, trading_mode): logger.info(f'Deleting existing data for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.') @@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: pair=pair, new_pairs_days=new_pairs_days, timerange=timerange, - data_handler=data_handler) + data_handler=data_handler, + trading_mode=trading_mode) return pairs_not_available @@ -521,7 +525,9 @@ def download_data_main(config: Config) -> None: pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_trades']) + erase=bool(config.get('erase')), data_format=config['dataformat_trades'], + trading_mode=config.get('trading_mode', TradingMode.SPOT), + ) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( From 66e43f2fe86e9948ce800e61b8362788e742c8f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:17:45 +0100 Subject: [PATCH 39/59] Adjust tests for new arguments --- tests/data/test_datahandler.py | 24 ++++++------- tests/data/test_history.py | 41 ++++++++++++----------- tests/data/test_trade_converter_kraken.py | 3 +- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index a0a37c393..1217c35ad 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ): def test_jsondatahandler_trades_load(testdatadir, caplog): dh = JsonGzDataHandler(testdatadir) logmsg = "Old trades format detected - converting" - dh.trades_load('XRP/ETH') + dh.trades_load('XRP/ETH', TradingMode.SPOT) assert not log_has(logmsg, caplog) # Test conversation is happening - dh.trades_load('XRP/OLD') + dh.trades_load('XRP/OLD', TradingMode.SPOT) assert log_has(logmsg, caplog) @@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected): def test_hdf5datahandler_trades_load(testdatadir): dh = get_datahandler(testdatadir, 'hdf5') - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades1, DataFrame) assert trades1.empty # data goes from 2019-10-11 - 2019-10-13 timerange = TimeRange.parse_timerange('20191011-20191012') - trades2 = dh._trades_load('XRP/ETH', timerange) + trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange) assert len(trades) > len(trades2) # Check that ID is None (If it's nan, it's wrong) assert trades2.iloc[0]['type'] is None @@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_load(testdatadir, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades.iloc[0]['timestamp'] == 1570752011620 assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000') assert trades.iloc[-1]['cost'] == 0.1986231 - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades1.empty @@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_store(testdatadir, tmp_path, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) dh1 = get_datahandler(tmp_path, datahandler) - dh1.trades_store('XRP/NEW', trades) + dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT) file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}' assert file.is_file() # Load trades back - trades_new = dh1.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT) assert_frame_equal(trades, trades_new, check_exact=True) assert len(trades_new) == len(trades) @@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = get_datahandler(testdatadir, datahandler) - assert not dh.trades_purge('UNITTEST/NONEXIST') + assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.trades_purge('UNITTEST/NONEXIST') + assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 1 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 106babd63..a3fe492b7 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl validate_backtest_data) from freqtrade.data.history.idatahandler import get_datahandler from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver @@ -168,21 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): assert fn == Path(expected_result + '.gz') -@pytest.mark.parametrize("pair,expected_result", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), - ("ETH/USDT:USDT", 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), - ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), - ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +@pytest.mark.parametrize("pair,trading_mode,expected_result", [ + ("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), + ("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), ]) -def test_json_pair_trades_filename(pair, expected_result): - fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) +def test_json_pair_trades_filename(pair, trading_mode, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') @@ -560,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes unavailable_pairs = refresh_backtest_trades_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"], datadir=testdatadir, - timerange=timerange, erase=True + timerange=timerange, erase=True, + trading_mode=TradingMode.SPOT, ) assert dl_mock.call_count == 2 @@ -585,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has("Current Amount of trades: 0", caplog) assert log_has("New Amount of trades: 6", caplog) assert ght_mock.call_count == 1 @@ -598,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[-3][0] // 1000) since_time2 = int(trades_history[-1][0] // 1000) timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='ETH/BTC', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 # Check this in seconds - since we had to convert to seconds above too. @@ -612,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad caplog.clear() assert not _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) file2 = tmp_path / 'XRP_ETH-trades.json.gz' @@ -624,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[0][0] // 1000) - 500 timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='XRP/ETH', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='XRP/ETH', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 diff --git a/tests/data/test_trade_converter_kraken.py b/tests/data/test_trade_converter_kraken.py index 91de303fb..ba9221e0a 100644 --- a/tests/data/test_trade_converter_kraken.py +++ b/tests/data/test_trade_converter_kraken.py @@ -6,6 +6,7 @@ import pytest from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from tests.conftest import EXMS, log_has, log_has_re, patch_exchange @@ -40,7 +41,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co assert dstfile.is_file() dh = get_datahandler(tmp_path, 'feather') - trades = dh.trades_load('BCH_EUR') + trades = dh.trades_load('BCH_EUR', TradingMode.SPOT) assert len(trades) == 340 assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56, From 5e7868a28dbadf96cefd27f280cd93016e0de6d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:22:41 +0100 Subject: [PATCH 40/59] Remove block from download-trades for futures --- freqtrade/data/history/history_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3f9468f7a..208859cd3 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -520,8 +520,6 @@ def download_data_main(config: Config) -> None: # Start downloading try: if config.get('download_trades'): - if config.get('trading_mode') == 'futures': - raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], From 09d763b604453ac702bc2d6e5d9adfbc3947fd69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:23:03 +0100 Subject: [PATCH 41/59] convert-trades should do proper pair expansion to support regex in pairlists --- freqtrade/commands/data_commands.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index d3600e3ef..b183d403b 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -11,6 +11,7 @@ from freqtrade.data.history import download_data_main from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util.migrations import migrate_data @@ -62,10 +63,17 @@ def start_convert_trades(args: Dict[str, Any]) -> None: for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) + available_pairs = [ + p for p in exchange.get_markets( + tradable_only=True, active_only=not config.get('include_inactive') + ).keys() + ] + + expanded_pairs = dynamic_expand_pairlist(config, available_pairs) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config.get('pairs', []), timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], From 99da6f70c2959f75e66850237086f20b8de007c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:27:54 +0100 Subject: [PATCH 42/59] Fix failing test due to new approach for convert-trades --- tests/commands/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1ab9d2202..2252ff9f4 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -837,10 +837,11 @@ def test_download_data_data_invalid(mocker): start_download_data(pargs) -def test_start_convert_trades(mocker, caplog): +def test_start_convert_trades(mocker): convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) + mocker.patch(f'{EXMS}.get_markets') mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={})) args = [ "trades-to-ohlcv", From 7ed7ed4081a1dedaecd025678268a71e0543f980 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 15:38:36 +0100 Subject: [PATCH 43/59] Accept trading-mode for trades-to-ohlcv command --- freqtrade/commands/arguments.py | 3 ++- freqtrade/data/converter/trade_converter.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index f72164675..191f07910 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase", ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"] -ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades", + "trading_mode"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 0e5050a05..1c8327ec3 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -99,8 +99,6 @@ def convert_trades_to_ohlcv( from freqtrade.data.history.idatahandler import get_datahandler data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) - if not pairs: - pairs = data_handler_trades.trades_get_pairs(datadir) logger.info(f"About to convert pairs: '{', '.join(pairs)}', " f"intervals: '{', '.join(timeframes)}' to {datadir}") From fcb16098d86779b17cdc889553d653547b5b3a51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 15:40:43 +0100 Subject: [PATCH 44/59] Reduce Error level when converting trades --- freqtrade/data/converter/trade_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 1c8327ec3..682430994 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -114,7 +114,7 @@ def convert_trades_to_ohlcv( # Store ohlcv data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) except ValueError: - logger.exception(f'Could not convert {pair} to OHLCV.') + logger.warning(f'Could not convert {pair} to OHLCV.') def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): From b8a1089592d2b6087fbd6dace25b2211d39cc93d Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 3 Mar 2024 12:23:18 +0100 Subject: [PATCH 45/59] fix: try plotting as much info in xgboost tensorboard as possible --- freqtrade/freqai/tensorboard/tensorboard.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/tensorboard/tensorboard.py b/freqtrade/freqai/tensorboard/tensorboard.py index 46bf8dc61..3ac58a117 100644 --- a/freqtrade/freqai/tensorboard/tensorboard.py +++ b/freqtrade/freqai/tensorboard/tensorboard.py @@ -46,10 +46,8 @@ class TensorBoardCallback(BaseTensorBoardCallback): for data, metric in evals_log.items(): for metric_name, log in metric.items(): score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] - if data == "train": - self.writer.add_scalar("train_loss", score, epoch) - else: - self.writer.add_scalar("valid_loss", score, epoch) + key = self._get_key(data, metric_name) + self.writer.add_scalar(f"{key}_loss", score, epoch) return False From 093a093bd513111dce24c200fc2636cb7e6185c1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 3 Mar 2024 12:38:51 +0100 Subject: [PATCH 46/59] fix: use data metric_name directly --- freqtrade/freqai/tensorboard/tensorboard.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqai/tensorboard/tensorboard.py b/freqtrade/freqai/tensorboard/tensorboard.py index 3ac58a117..d91c70c81 100644 --- a/freqtrade/freqai/tensorboard/tensorboard.py +++ b/freqtrade/freqai/tensorboard/tensorboard.py @@ -46,8 +46,7 @@ class TensorBoardCallback(BaseTensorBoardCallback): for data, metric in evals_log.items(): for metric_name, log in metric.items(): score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] - key = self._get_key(data, metric_name) - self.writer.add_scalar(f"{key}_loss", score, epoch) + self.writer.add_scalar(f"{data}-{metric_name}", score, epoch) return False From ed8469f23ac32fe0ca6e6d6eb06b9cd7e4f07a70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 13:12:42 +0100 Subject: [PATCH 47/59] use trading_mode to determine trades file location --- freqtrade/data/history/idatahandler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bcb31a7c8..fbaded640 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -277,9 +277,8 @@ class IDataHandler(ABC): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path: pair_s = misc.pair_to_filename(pair) - if ':' in pair: + if trading_mode == TradingMode.FUTURES: # Futures pair ... - # TODO: this should not rely on ";" in the pairname. datadir = datadir.joinpath('futures') filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') From a948796ef74449952a4f57268304f616e8a6aa33 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 3 Mar 2024 15:47:19 +0100 Subject: [PATCH 48/59] fix: manually add train eval since xgboost does not expose this information by default --- .../freqai/prediction_models/XGBoostRegressor.py | 11 +++++++++-- freqtrade/freqai/tensorboard/tensorboard.py | 7 ++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index f1a2474da..f3de6653b 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -36,8 +36,15 @@ class XGBoostRegressor(BaseRegressionModel): eval_set = None eval_weights = None else: - eval_set = [(data_dictionary["test_features"], data_dictionary["test_labels"])] - eval_weights = [data_dictionary['test_weights']] + eval_set = [ + (data_dictionary["test_features"], + data_dictionary["test_labels"]), + (X, y) + ] + eval_weights = [ + data_dictionary['test_weights'], + data_dictionary['train_weights'] + ] sample_weight = data_dictionary["train_weights"] diff --git a/freqtrade/freqai/tensorboard/tensorboard.py b/freqtrade/freqai/tensorboard/tensorboard.py index d91c70c81..3ad896108 100644 --- a/freqtrade/freqai/tensorboard/tensorboard.py +++ b/freqtrade/freqai/tensorboard/tensorboard.py @@ -43,10 +43,11 @@ class TensorBoardCallback(BaseTensorBoardCallback): if not evals_log: return False - for data, metric in evals_log.items(): - for metric_name, log in metric.items(): + evals = ["validation", "train"] + for metric, eval in zip(evals_log.items(), evals): + for metric_name, log in metric[1].items(): score = log[-1][0] if isinstance(log[-1], tuple) else log[-1] - self.writer.add_scalar(f"{data}-{metric_name}", score, epoch) + self.writer.add_scalar(f"{eval}-{metric_name}", score, epoch) return False From 07bf19a990ad8342014c558f81a729a584b49585 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:17:13 +0000 Subject: [PATCH 49/59] Bump pypa/gh-action-pypi-publish from 1.8.11 to 1.8.12 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.11 to 1.8.12. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.11...v1.8.12) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba55eed04..e0587525e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -482,12 +482,12 @@ jobs: path: dist - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.8.11 + uses: pypa/gh-action-pypi-publish@v1.8.12 with: repository-url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.11 + uses: pypa/gh-action-pypi-publish@v1.8.12 deploy-docker: From 7ca3032d515bdf2bf5d6b48440c16f053216629d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:22:32 +0000 Subject: [PATCH 50/59] Bump the pytest group with 1 update Bumps the pytest group with 1 update: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 8.0.2 to 8.1.0 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.2...8.1.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor dependency-group: pytest ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 77d981087..1014648ce 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.2.2 mypy==1.8.0 pre-commit==3.6.2 -pytest==8.0.2 +pytest==8.1.0 pytest-asyncio==0.23.5 pytest-cov==4.1.0 pytest-mock==3.12.0 From 21709204eb8fef248870704cb1a1793205e641bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:22:37 +0000 Subject: [PATCH 51/59] Bump time-machine from 2.13.0 to 2.14.0 Bumps [time-machine](https://github.com/adamchainz/time-machine) from 2.13.0 to 2.14.0. - [Changelog](https://github.com/adamchainz/time-machine/blob/main/CHANGELOG.rst) - [Commits](https://github.com/adamchainz/time-machine/compare/2.13.0...2.14.0) --- updated-dependencies: - dependency-name: time-machine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 77d981087..a05aa478f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ pytest-random-order==1.1.1 pytest-xdist==3.5.0 isort==5.13.2 # For datetime mocking -time-machine==2.13.0 +time-machine==2.14.0 # Convert jupyter notebooks to markdown documents nbconvert==7.16.1 From 062376f5735987e4b57f18e860b9685c41ed4682 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:22:41 +0000 Subject: [PATCH 52/59] Bump mkdocs-material from 9.5.11 to 9.5.12 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.11 to 9.5.12. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.11...9.5.12) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index cbb81b6b2..55a2c11aa 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.5.2 mkdocs==1.5.3 -mkdocs-material==9.5.11 +mkdocs-material==9.5.12 mdx_truly_sane_lists==1.3 pymdown-extensions==10.7 jinja2==3.1.3 From c3f9b16c8456551f7c700e017479379794c58e73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:22:46 +0000 Subject: [PATCH 53/59] Bump rich from 13.7.0 to 13.7.1 Bumps [rich](https://github.com/Textualize/rich) from 13.7.0 to 13.7.1. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.7.0...v13.7.1) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0477751cd..90166b267 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ pycoingecko==3.1.0 jinja2==3.1.3 tables==3.9.1 joblib==1.3.2 -rich==13.7.0 +rich==13.7.1 pyarrow==15.0.0; platform_machine != 'armv7l' # find first, C search in arrays From b8c16fb889244a0445b39f014735046bcad2cd7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:23:02 +0000 Subject: [PATCH 54/59] Bump python-dateutil from 2.8.2 to 2.9.0.post0 Bumps [python-dateutil](https://github.com/dateutil/dateutil) from 2.8.2 to 2.9.0.post0. - [Release notes](https://github.com/dateutil/dateutil/releases) - [Changelog](https://github.com/dateutil/dateutil/blob/master/NEWS) - [Commits](https://github.com/dateutil/dateutil/compare/2.8.2...2.9.0.post0) --- updated-dependencies: - dependency-name: python-dateutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0477751cd..449b5be28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,7 +49,7 @@ colorama==0.4.6 questionary==2.0.1 prompt-toolkit==3.0.36 # Extensions to datetime library -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 pytz==2024.1 #Futures From ec17b5523c98c3c167473838424978151707b14c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:23:10 +0000 Subject: [PATCH 55/59] Bump cachetools from 5.3.2 to 5.3.3 Bumps [cachetools](https://github.com/tkem/cachetools) from 5.3.2 to 5.3.3. - [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tkem/cachetools/compare/v5.3.2...v5.3.3) --- updated-dependencies: - dependency-name: cachetools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0477751cd..e94313984 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ python-telegram-bot==20.8 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 arrow==1.3.0 -cachetools==5.3.2 +cachetools==5.3.3 requests==2.31.0 urllib3==2.2.1 jsonschema==4.21.1 From d6ae63ac484cefccbef885219000eefa8326a7da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:23:16 +0000 Subject: [PATCH 56/59] Bump python-rapidjson from 1.14 to 1.16 Bumps [python-rapidjson](https://github.com/python-rapidjson/python-rapidjson) from 1.14 to 1.16. - [Changelog](https://github.com/python-rapidjson/python-rapidjson/blob/master/CHANGES.rst) - [Commits](https://github.com/python-rapidjson/python-rapidjson/compare/v1.14...v1.16) --- updated-dependencies: - dependency-name: python-rapidjson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0477751cd..86deabc3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ pyarrow==15.0.0; platform_machine != 'armv7l' py_find_1st==1.1.6 # Load ticker files 30% faster -python-rapidjson==1.14 +python-rapidjson==1.16 # Properly format api responses orjson==3.9.15 From 8c0ba2a69aff9acac58669880b23e98c08a22cae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 03:23:27 +0000 Subject: [PATCH 57/59] Bump ccxt from 4.2.51 to 4.2.58 Bumps [ccxt](https://github.com/ccxt/ccxt) from 4.2.51 to 4.2.58. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/4.2.51...4.2.58) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0477751cd..8e862275a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.4 pandas==2.2.1 pandas-ta==0.3.14b -ccxt==4.2.51 +ccxt==4.2.58 cryptography==42.0.5 aiohttp==3.9.3 SQLAlchemy==2.0.27 From 9ad60643f5d7df7c40585f870fd244462e824172 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 05:46:23 +0000 Subject: [PATCH 58/59] Bump ruff from 0.2.2 to 0.3.0 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.2.2 to 0.3.0. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.2.2...v0.3.0) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 31c8a4f8c..562841375 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.2.2 +ruff==0.3.0 mypy==1.8.0 pre-commit==3.6.2 pytest==8.1.0 From 99c8be4c30e931d64bc96f3f471daa953aff2e93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 07:59:10 +0000 Subject: [PATCH 59/59] Bump pydantic from 2.6.2 to 2.6.3 Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.6.2 to 2.6.3. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.6.2...v2.6.3) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 36ca1b76f..44e8e2ccb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ sdnotify==0.3.2 # API Server fastapi==0.110.0 -pydantic==2.6.2 +pydantic==2.6.3 uvicorn==0.27.1 pyjwt==2.8.0 aiofiles==23.2.1