Merge branch 'freqtrade:develop' into delist

This commit is contained in:
Stefano
2025-09-22 13:38:03 +09:00
committed by GitHub
24 changed files with 2528 additions and 1464 deletions

View File

@@ -45,7 +45,6 @@ jobs:
python-version: ${{ matrix.python-version }}
cache-dependency-glob: "requirements**.txt"
cache-suffix: "${{ matrix.python-version }}"
prune-cache: false
- name: Installation - *nix
run: |
@@ -166,30 +165,11 @@ jobs:
python-version: ${{ matrix.python-version }}
cache-dependency-glob: "requirements**.txt"
cache-suffix: "${{ matrix.python-version }}"
prune-cache: false
- name: Installation - macOS (Brew)
run: |
# brew update
# TODO: Should be the brew upgrade
# homebrew fails to update python due to unlinking failures
# https://github.com/actions/runner-images/issues/6817
rm /usr/local/bin/2to3 || true
rm /usr/local/bin/2to3-3.11 || true
rm /usr/local/bin/2to3-3.12 || true
rm /usr/local/bin/idle3 || true
rm /usr/local/bin/idle3.11 || true
rm /usr/local/bin/idle3.12 || true
rm /usr/local/bin/pydoc3 || true
rm /usr/local/bin/pydoc3.11 || true
rm /usr/local/bin/pydoc3.12 || true
rm /usr/local/bin/python3 || true
rm /usr/local/bin/python3.11 || true
rm /usr/local/bin/python3.12 || true
rm /usr/local/bin/python3-config || true
rm /usr/local/bin/python3.11-config || true
rm /usr/local/bin/python3.12-config || true
brew install libomp
- name: Installation (python)
@@ -278,7 +258,6 @@ jobs:
python-version: ${{ matrix.python-version }}
cache-dependency-glob: "requirements**.txt"
cache-suffix: "${{ matrix.python-version }}"
prune-cache: false
- name: Installation
run: |
@@ -428,7 +407,6 @@ jobs:
python-version: "3.12"
cache-dependency-glob: "requirements**.txt"
cache-suffix: "3.12"
prune-cache: false
- name: Installation - *nix
run: |

View File

@@ -10,6 +10,7 @@ usage: freqtrade backtesting [-h] [-v] [--no-color] [--logfile FILE] [-V]
[--stake-amount STAKE_AMOUNT] [--fee FLOAT]
[-p PAIRS [PAIRS ...]] [--eps]
[--enable-protections]
[--enable-dynamic-pairlist]
[--dry-run-wallet DRY_RUN_WALLET]
[--timeframe-detail TIMEFRAME_DETAIL]
[--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
@@ -44,9 +45,14 @@ options:
Allow buying the same pair multiple times (position
stacking).
--enable-protections, --enableprotections
Enable protections for backtesting.Will slow
Enable protections for backtesting. Will slow
backtesting down by a considerable amount, but will
include configured protections
--enable-dynamic-pairlist
Enables dynamic pairlist refreshes in backtesting. The
pairlist will be generated for each new candle if
you're using a pairlist handler that supports this
feature, for example, ShuffleFilter.
--dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET
Starting balance, used for backtesting / hyperopt and
dry-runs.

View File

@@ -44,7 +44,7 @@ options:
Allow buying the same pair multiple times (position
stacking).
--enable-protections, --enableprotections
Enable protections for backtesting.Will slow
Enable protections for backtesting. Will slow
backtesting down by a considerable amount, but will
include configured protections
--dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET

View File

@@ -11,6 +11,7 @@ usage: freqtrade lookahead-analysis [-h] [-v] [--no-color] [--logfile FILE]
[--stake-amount STAKE_AMOUNT]
[--fee FLOAT] [-p PAIRS [PAIRS ...]]
[--enable-protections]
[--enable-dynamic-pairlist]
[--dry-run-wallet DRY_RUN_WALLET]
[--timeframe-detail TIMEFRAME_DETAIL]
[--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
@@ -44,9 +45,14 @@ options:
Limit command to these pairs. Pairs are space-
separated.
--enable-protections, --enableprotections
Enable protections for backtesting.Will slow
Enable protections for backtesting. Will slow
backtesting down by a considerable amount, but will
include configured protections
--enable-dynamic-pairlist
Enables dynamic pairlist refreshes in backtesting. The
pairlist will be generated for each new candle if
you're using a pairlist handler that supports this
feature, for example, ShuffleFilter.
--dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET
Starting balance, used for backtesting / hyperopt and
dry-runs.

View File

@@ -70,7 +70,16 @@ Things you can change (among others):
![FreqUI - Settings view](assets/frequi-settings-dark.png#only-dark)
![FreqUI - Settings view](assets/frequi-settings-light.png#only-light)
## Backtesting
## Webserver mode
when freqtrade is started in [webserver mode](utils.md#webserver-mode) (freqtrade started with `freqtrade webserver`), the webserver will start in a special mode allowing for additional features, for example:
* Downloading data
* Testing pairlists
* [Backtesting strategies](#backtesting)
* ... to be expanded
### Backtesting
When freqtrade is started in [webserver mode](utils.md#webserver-mode) (freqtrade started with `freqtrade webserver`), the backtesting view becomes available.
This view allows you to backtest strategies and visualize the results.

View File

@@ -39,7 +39,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
* [`VolatilityFilter`](#volatilityfilter)
!!! Tip "Testing pairlists"
Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
Pairlist configurations can be quite tricky to get right. Best use freqUI in [webserver mode](freq-ui.md#webserver-mode) or the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your Pairlist configuration quickly.
#### Static Pair List

View File

@@ -1,6 +1,6 @@
markdown==3.9
mkdocs==1.6.1
mkdocs-material==9.6.19
mkdocs-material==9.6.20
mdx_truly_sane_lists==1.3
pymdown-extensions==10.16.1
jinja2==3.1.6

View File

@@ -49,6 +49,7 @@ ARGS_BACKTEST = [
*ARGS_COMMON_OPTIMIZE,
"position_stacking",
"enable_protections",
"enable_dynamic_pairlist",
"dry_run_wallet",
"timeframe_detail",
"strategy_list",

View File

@@ -184,12 +184,20 @@ AVAILABLE_CLI_OPTIONS = {
"enable_protections": Arg(
"--enable-protections",
"--enableprotections",
help="Enable protections for backtesting."
help="Enable protections for backtesting. "
"Will slow backtesting down by a considerable amount, but will include "
"configured protections",
action="store_true",
default=False,
),
"enable_dynamic_pairlist": Arg(
"--enable-dynamic-pairlist",
help="Enables dynamic pairlist refreshes in backtesting. "
"The pairlist will be generated for each new candle if you're using a "
"pairlist handler that supports this feature, for example, ShuffleFilter.",
action="store_true",
default=False,
),
"strategy_list": Arg(
"--strategy-list",
help="Provide a space-separated list of strategies to backtest. "

View File

@@ -259,7 +259,13 @@ class Configuration:
self._args_to_config(
config,
argname="enable_protections",
logstring="Parameter --enable-protections detected, enabling Protections. ...",
logstring="Parameter --enable-protections detected, enabling Protections ...",
)
self._args_to_config(
config,
argname="enable_dynamic_pairlist",
logstring="Parameter --enable-dynamic-pairlist detected, enabling dynamic pairlist ...",
)
if self.args.get("max_open_trades"):

View File

@@ -181,7 +181,6 @@ def trim_dataframes(
def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
"""
TODO: This should get a dedicated test
Gets order book list, returns dataframe with below format per suggested by creslin
-------------------------------------------------------------------
b_sum b_size bids asks a_size a_sum

File diff suppressed because it is too large Load Diff

View File

@@ -692,12 +692,13 @@ class Exchange:
# Reload async markets, then assign them to sync api
retrier(self._load_async_markets, retries=retries)(reload=True)
self._markets = self._api_async.markets
self._api.set_markets(self._api_async.markets, self._api_async.currencies)
self._api.set_markets_from_exchange(self._api_async)
# Assign options array, as it contains some temporary information from the exchange.
# TODO: investigate with ccxt if it's safe to remove `.options`
self._api.options = self._api_async.options
if self._exchange_ws:
# Set markets to avoid reloading on websocket api
self._ws_async.set_markets(self._api.markets, self._api.currencies)
self._ws_async.set_markets_from_exchange(self._api_async)
self._ws_async.options = self._api.options
self._last_markets_refresh = dt_ts()

View File

@@ -211,6 +211,7 @@ class Backtesting:
self._can_short = self.trading_mode != TradingMode.SPOT
self._position_stacking: bool = self.config.get("position_stacking", False)
self.enable_protections: bool = self.config.get("enable_protections", False)
self.dynamic_pairlist: bool = self.config.get("enable_dynamic_pairlist", False)
migrate_data(config, self.exchange)
self.init_backtest()
@@ -1584,6 +1585,11 @@ class Backtesting:
for current_time in self._time_generator(start_date, end_date):
# Loop for each main candle.
self.check_abort()
if self.dynamic_pairlist and self.pairlists:
self.pairlists.refresh_pairlist()
pairs = self.pairlists.whitelist
# Reset open trade count for this candle
# Critical to avoid exceeding max_open_trades in backtesting
# when timeframe-detail is used and trades close within the opening candle.

View File

@@ -93,6 +93,8 @@ class ShuffleFilter(IPairList):
return pairlist_new
# Shuffle is done inplace
self._random.shuffle(pairlist)
if self._config.get("runmode") in (RunMode.LIVE, RunMode.DRY_RUN):
self.__pairlist_cache[pairlist_bef] = pairlist
return pairlist

View File

@@ -7,6 +7,9 @@ Provides pair white list as it configured in config
import logging
from copy import deepcopy
from cachetools import LRUCache
from freqtrade.enums import RunMode
from freqtrade.exchange.exchange_types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter, SupportsBacktesting
@@ -22,6 +25,8 @@ class StaticPairList(IPairList):
super().__init__(*args, **kwargs)
self._allow_inactive = self._pairlistconfig.get("allow_inactive", False)
# Pair cache - only used for optimize modes
self._bt_pair_cache: LRUCache = LRUCache(maxsize=1)
@property
def needstickers(self) -> bool:
@@ -60,15 +65,23 @@ class StaticPairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: List of pairs
"""
pairlist = self._bt_pair_cache.get("pairlist")
if not pairlist:
wl = self.verify_whitelist(
self._config["exchange"]["pair_whitelist"], logger.info, keep_invalid=True
)
if self._allow_inactive:
return wl
pairlist = wl
else:
# Avoid implicit filtering of "verify_whitelist" to keep
# proper warnings in the log
return self._whitelist_for_active_markets(wl)
pairlist = self._whitelist_for_active_markets(wl)
if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
self._bt_pair_cache["pairlist"] = pairlist.copy()
return pairlist
def filter_pairlist(self, pairlist: list[str], tickers: Tickers) -> list[str]:
"""

View File

@@ -5,7 +5,7 @@ PairList manager class
import logging
from functools import partial
from cachetools import TTLCache, cached
from cachetools import LRUCache, TTLCache, cached
from freqtrade.constants import Config, ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
@@ -56,6 +56,7 @@ class PairListManager(LoggingMixin):
)
self._check_backtest()
self._not_expiring_cache: LRUCache = LRUCache(maxsize=1)
refresh_period = config.get("pairlist_refresh_period", 3600)
LoggingMixin.__init__(self, logger, refresh_period)
@@ -109,7 +110,15 @@ class PairListManager(LoggingMixin):
@property
def expanded_blacklist(self) -> list[str]:
"""The expanded blacklist (including wildcard expansion)"""
return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
eblacklist = self._not_expiring_cache.get("eblacklist")
if not eblacklist:
eblacklist = expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
if self._config["runmode"] in (RunMode.BACKTEST, RunMode.HYPEROPT):
self._not_expiring_cache["eblacklist"] = eblacklist.copy()
return eblacklist
@property
def name_list(self) -> list[str]:
@@ -157,6 +166,7 @@ class PairListManager(LoggingMixin):
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
:return: pairlist - blacklisted pairs
"""
if self._blacklist:
try:
blacklist = self.expanded_blacklist
except ValueError as err:

View File

@@ -360,11 +360,9 @@ class Telegram(RPCHandler):
await asyncio.sleep(2)
if self._app.updater:
await self._app.updater.start_polling(
bootstrap_retries=-1,
bootstrap_retries=10,
timeout=20,
# read_latency=60, # Assumed transmission latency
drop_pending_updates=True,
# stop_signals=[], # Necessary as we don't run on the main thread
)
while True:
await asyncio.sleep(10)

View File

@@ -29,7 +29,7 @@ classifiers = [
dependencies = [
# from requirements.txt
"ccxt>=4.4.87",
"ccxt>=4.5.4",
"SQLAlchemy>=2.0.6",
"python-telegram-bot>=20.1",
"humanize>=4.0.0",

View File

@@ -6,13 +6,13 @@
-r requirements-freqai-rl.txt
-r docs/requirements-docs.txt
ruff==0.13.0
ruff==0.13.1
mypy==1.18.1
pre-commit==4.3.0
pytest==8.4.2
pytest-asyncio==1.2.0
pytest-cov==7.0.0
pytest-mock==3.15.0
pytest-mock==3.15.1
pytest-random-order==1.2.0
pytest-timeout==2.4.0
pytest-xdist==3.8.0

View File

@@ -7,7 +7,7 @@ ft-pandas-ta==0.3.15
ta-lib==0.6.7
technical==1.5.3
ccxt==4.5.4
ccxt==4.5.5
cryptography==45.0.7
aiohttp==3.12.15
SQLAlchemy==2.0.43
@@ -38,12 +38,12 @@ orjson==3.11.3
sdnotify==0.3.2
# API Server
fastapi==0.116.1
fastapi==0.117.1
pydantic==2.11.9
uvicorn==0.35.0
uvicorn==0.36.0
pyjwt==2.10.1
aiofiles==24.1.0
psutil==7.0.0
psutil==7.1.0
# Building config files interactively
questionary==2.1.1

View File

@@ -14,6 +14,7 @@ from freqtrade.data.converter import (
convert_trades_to_ohlcv,
ohlcv_fill_up_missing_data,
ohlcv_to_dataframe,
order_book_to_dataframe,
reduce_dataframe_footprint,
trades_df_remove_duplicates,
trades_dict_to_list,
@@ -588,3 +589,77 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
candle_type=CandleType.SPOT,
)
assert log_has(msg, caplog)
def test_order_book_to_dataframe():
bids = [
[100.0, 5.0],
[99.5, 3.0],
[99.0, 2.0],
]
asks = [
[100.5, 4.0],
[101.0, 6.0],
[101.5, 1.0],
]
result = order_book_to_dataframe(bids, asks)
assert isinstance(result, pd.DataFrame)
expected_columns = ["b_sum", "b_size", "bids", "asks", "a_size", "a_sum"]
assert result.columns.tolist() == expected_columns
assert len(result) == max(len(bids), len(asks))
assert result["bids"].tolist() == [100.0, 99.5, 99.0]
assert result["b_size"].tolist() == [5.0, 3.0, 2.0]
assert result["b_sum"].tolist() == [5.0, 8.0, 10.0]
assert result["asks"].tolist() == [100.5, 101.0, 101.5]
assert result["a_size"].tolist() == [4.0, 6.0, 1.0]
assert result["a_sum"].tolist() == [4.0, 10.0, 11.0]
def test_order_book_to_dataframe_empty():
bids = []
asks = []
result = order_book_to_dataframe(bids, asks)
assert isinstance(result, pd.DataFrame)
expected_columns = ["b_sum", "b_size", "bids", "asks", "a_size", "a_sum"]
assert result.columns.tolist() == expected_columns
# Empty input should result in empty dataframe
assert len(result) == 0
def test_order_book_to_dataframe_unequal_lengths():
bids = [
[100.0, 5.0],
[99.5, 3.0],
[99.0, 2.0],
[98.5, 1.0],
]
asks = [
[100.5, 4.0],
[101.0, 6.0],
]
result = order_book_to_dataframe(bids, asks)
assert len(result) == max(len(bids), len(asks))
assert len(result) == 4
assert result["bids"].tolist() == [100.0, 99.5, 99.0, 98.5]
assert result["b_size"].tolist() == [5.0, 3.0, 2.0, 1.0]
assert result["b_sum"].tolist() == [5.0, 8.0, 10.0, 11.0]
assert result["asks"].tolist()[:2] == [100.5, 101.0]
# NA for missing asks
assert pd.isna(result["asks"].iloc[2])
assert pd.isna(result["asks"].iloc[3])
assert result["a_size"].tolist()[:2] == [4.0, 6.0]
assert result["a_sum"].tolist()[:2] == [4.0, 10.0]

View File

@@ -27,7 +27,7 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename,
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.persistence import LocalTrade, Trade
from freqtrade.resolvers import StrategyResolver
from freqtrade.util.datetime_helpers import dt_utc
from freqtrade.util import dt_now, dt_utc
from tests.conftest import (
CURRENT_TEST_STRATEGY,
EXMS,
@@ -2715,3 +2715,75 @@ def test_get_backtest_metadata_filename():
filename = "backtest_results_zip.zip"
expected = Path("backtest_results_zip.meta.json")
assert get_backtest_metadata_filename(filename) == expected
@pytest.mark.parametrize("dynamic_pairlist", [True, False])
def test_time_pair_generator_refresh_pairlist(mocker, default_conf, dynamic_pairlist):
patch_exchange(mocker)
default_conf["enable_dynamic_pairlist"] = dynamic_pairlist
backtesting = Backtesting(default_conf)
backtesting._set_strategy(backtesting.strategylist[0])
assert backtesting.dynamic_pairlist == dynamic_pairlist
refresh_mock = mocker.patch(
"freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist"
)
# Simulate 2 candles
start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC)
end_date = start_date + timedelta(minutes=10)
pairs = default_conf["exchange"]["pair_whitelist"]
data = {pair: [] for pair in pairs}
# Simulate backtest loop
list(backtesting.time_pair_generator(start_date, end_date, pairs, data))
if dynamic_pairlist:
assert refresh_mock.call_count == 2
else:
assert refresh_mock.call_count == 0
@pytest.mark.parametrize("dynamic_pairlist", [True, False])
def test_time_pair_generator_open_trades_first(mocker, default_conf, dynamic_pairlist):
patch_exchange(mocker)
default_conf["enable_dynamic_pairlist"] = dynamic_pairlist
backtesting = Backtesting(default_conf)
backtesting._set_strategy(backtesting.strategylist[0])
assert backtesting.dynamic_pairlist == dynamic_pairlist
pairs = ["XRP/BTC", "LTC/BTC", "NEO/BTC", "ETH/BTC"]
# Simulate open trades
trades = [
LocalTrade(pair="XRP/BTC", open_date=dt_now(), amount=1, open_rate=1),
LocalTrade(pair="NEO/BTC", open_date=dt_now(), amount=1, open_rate=1),
]
LocalTrade.bt_trades_open = trades
LocalTrade.bt_trades_open_pp = {
"XRP/BTC": [trades[0]],
"NEO/BTC": [trades[1]],
"LTC/BTC": [],
"ETH/BTC": [],
}
start_date = datetime(2025, 1, 1, 0, 0, tzinfo=UTC)
end_date = start_date + timedelta(minutes=5)
dummy_row = (end_date, 1.0, 1.1, 0.9, 1.0, 0, 0, 0, 0, None, None)
data = {pair: [dummy_row] for pair in pairs}
def mock_refresh(self):
# Simulate shuffle
self._whitelist = pairs[::-1] # ['ETH/BTC', 'NEO/BTC', 'LTC/BTC', 'XRP/BTC']
mocker.patch("freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist", mock_refresh)
processed_pairs = []
for _, pair, _, _, _ in backtesting.time_pair_generator(start_date, end_date, pairs, data):
processed_pairs.append(pair)
# Open trades first in both cases
if dynamic_pairlist:
assert processed_pairs == ["XRP/BTC", "NEO/BTC", "ETH/BTC", "LTC/BTC"]
else:
assert processed_pairs == ["XRP/BTC", "NEO/BTC", "LTC/BTC", "ETH/BTC"]

View File

@@ -1274,27 +1274,37 @@ def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None:
{"method": "StaticPairList"},
{"method": "ShuffleFilter", "seed": 43},
]
whitelist_conf["runmode"] = "backtest"
whitelist_conf["runmode"] = RunMode.BACKTEST
exchange = get_patched_exchange(mocker, whitelist_conf)
plm = PairListManager(exchange, whitelist_conf)
assert log_has("Backtesting mode detected, applying seed value: 43", caplog)
plm.refresh_pairlist()
pl1 = deepcopy(plm.whitelist)
plm.refresh_pairlist()
assert plm.whitelist != pl1
assert set(plm.whitelist) == set(pl1)
caplog.clear()
whitelist_conf["runmode"] = RunMode.DRY_RUN
plm = PairListManager(exchange, whitelist_conf)
assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
assert log_has("Live mode detected, not applying seed.", caplog)
with time_machine.travel("2021-09-01 05:01:00 +00:00") as t:
plm.refresh_pairlist()
pl1 = deepcopy(plm.whitelist)
plm.refresh_pairlist()
assert plm.whitelist == pl1
target = plm._pairlist_handlers[1]._random
shuffle_mock = mocker.patch.object(target, "shuffle", wraps=target.shuffle)
t.shift(timedelta(minutes=10))
plm.refresh_pairlist()
assert plm.whitelist != pl1
caplog.clear()
whitelist_conf["runmode"] = RunMode.DRY_RUN
plm = PairListManager(exchange, whitelist_conf)
assert not log_has("Backtesting mode detected, applying seed value: 42", caplog)
assert log_has("Live mode detected, not applying seed.", caplog)
assert shuffle_mock.call_count == 1
assert set(plm.whitelist) == set(pl1)
@pytest.mark.usefixtures("init_persistence")