diff --git a/Dockerfile b/Dockerfile index e5a33df87..a1205f219 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.7-slim-bookworm as base +FROM python:3.11.8-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 4cb8f5fea..1165f305c 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.11.7-slim-bookworm as base +FROM python:3.11.8-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index aca3da72a..cbb81b6b2 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.5.2 mkdocs==1.5.3 -mkdocs-material==9.5.9 +mkdocs-material==9.5.11 mdx_truly_sane_lists==1.3 pymdown-extensions==10.7 jinja2==3.1.3 diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 2292b7ed0..2f04e906e 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -791,7 +791,7 @@ Returning a value more than the above (so remaining stake_amount would become ne If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that. Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order. -!!! Warning +!!! Warning "Stoploss calculation" Stoploss is still calculated from the initial opening price, not averaged price. Regular stoploss rules still apply (cannot move down). @@ -801,6 +801,11 @@ Returning a value more than the above (so remaining stake_amount would become ne During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected. This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle. +!!! Warning "Performance with many position adjustments" + Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively. + Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage. + Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance. + ``` python from freqtrade.persistence import Trade diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 7c699d643..fa5d9214e 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2024.2-dev' +__version__ = '2024.3-dev' if 'dev' in __version__: from pathlib import Path diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index d22fd9e31..b118bd7e0 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -35,7 +35,7 @@ class HDF5DataHandler(IDataHandler): self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( - filename, key, mode='a', complevel=9, complib='blosc', + filename, key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['date'] ) @@ -110,7 +110,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_trades_key(pair) data.to_hdf( - self._pair_trades_filename(self._datadir, pair), key, + self._pair_trades_filename(self._datadir, pair), key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 9a02a7769..baa0c10a5 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -37,7 +37,7 @@ class JsonDataHandler(IDataHandler): self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int - _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 + _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000 # Reset index, select only appropriate columns and save as json _data.reset_index(drop=True).loc[:, self._columns].to_json( diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3f23f43ae..974f8124e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -128,8 +128,9 @@ class FreqtradeBot(LoggingMixin): self.update_funding_fees() self.wallets.update() - # TODO: This would be more efficient if scheduled in utc time, and performed at each - # TODO: funding interval, specified by funding_fee_times on the exchange classes + # This would be more efficient if scheduled in utc time, and performed at each + # funding interval, specified by funding_fee_times on the exchange classes + # However, this reduces the precision - and might therefore lead to problems. for time_slot in range(0, 24): for minutes in [1, 31]: t = str(time(time_slot, minutes, 2)) diff --git a/freqtrade/optimize/analysis/lookahead_helpers.py b/freqtrade/optimize/analysis/lookahead_helpers.py index 1d2b9db48..00f83a46b 100644 --- a/freqtrade/optimize/analysis/lookahead_helpers.py +++ b/freqtrade/optimize/analysis/lookahead_helpers.py @@ -107,9 +107,9 @@ class LookaheadAnalysisSubFunctions: csv_df = add_or_update_row(csv_df, new_row_data) # Fill NaN values with a default value (e.g., 0) - csv_df['total_signals'] = csv_df['total_signals'].fillna(0) - csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].fillna(0) - csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].fillna(0) + csv_df['total_signals'] = csv_df['total_signals'].astype(int).fillna(0) + csv_df['biased_entry_signals'] = csv_df['biased_entry_signals'].astype(int).fillna(0) + csv_df['biased_exit_signals'] = csv_df['biased_exit_signals'].astype(int).fillna(0) # Convert columns to integers csv_df['total_signals'] = csv_df['total_signals'].astype(int) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 493c7567f..8d16122ea 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -201,7 +201,7 @@ class Backtesting: self.prepare_backtest(False) - self.wallets = Wallets(self.config, self.exchange, log=False) + self.wallets = Wallets(self.config, self.exchange, is_backtest=True) self.progress = BTProgress() self.abort = False diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 47a13dcd8..47aab2a62 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -215,7 +215,7 @@ def _get_resample_from_period(period: str) -> str: # Weekly defaulting to Monday. return '1W-MON' if period == 'month': - return '1M' + return '1ME' raise ValueError(f"Period {period} is not supported.") diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2317ee1a9..6e8447d29 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1155,7 +1155,7 @@ class RPC: } if has_content: - dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000 + dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 # Move signal close to separate column when signal for easy plotting for sig_type in signals.keys(): if sig_type in dataframe.columns: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 0f41114ed..0d22feb36 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -36,9 +36,9 @@ class PositionWallet(NamedTuple): class Wallets: - def __init__(self, config: Config, exchange: Exchange, log: bool = True) -> None: + def __init__(self, config: Config, exchange: Exchange, is_backtest: bool = False) -> None: self._config = config - self._log = log + self._is_backtest = is_backtest self._exchange = exchange self._wallets: Dict[str, Wallet] = {} self._positions: Dict[str, PositionWallet] = {} @@ -78,11 +78,11 @@ class Wallets: _wallets = {} _positions = {} open_trades = Trade.get_trades_proxy(is_open=True) - # If not backtesting... - # TODO: potentially remove the ._log workaround to determine backtest mode. - if self._log: + if not self._is_backtest: + # Live / Dry-run mode tot_profit = Trade.get_total_closed_profit() else: + # Backtest mode tot_profit = LocalTrade.total_profit tot_profit += sum(trade.realized_profit for trade in open_trades) tot_in_trades = sum(trade.stake_amount for trade in open_trades) @@ -177,7 +177,7 @@ class Wallets: self._update_live() else: self._update_dry() - if self._log: + if not self._is_backtest: logger.info('Wallets synced.') self._last_wallet_refresh = dt_now() @@ -341,19 +341,19 @@ class Wallets: max_allowed_stake = min(max_allowed_stake, max_stake_amount - trade_amount) if min_stake_amount is not None and min_stake_amount > max_allowed_stake: - if self._log: + if not self._is_backtest: logger.warning("Minimum stake amount > available balance. " f"{min_stake_amount} > {max_allowed_stake}") return 0 if min_stake_amount is not None and stake_amount < min_stake_amount: - if self._log: + if not self._is_backtest: logger.info( f"Stake amount for pair {pair} is too small " f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}." ) if stake_amount * 1.3 < min_stake_amount: # Top-cap stake-amount adjustments to +30%. - if self._log: + if not self._is_backtest: logger.info( f"Adjusted stake amount for pair {pair} is more than 30% bigger than " f"the desired stake amount of ({stake_amount:.8f} * 1.3 = " @@ -363,7 +363,7 @@ class Wallets: stake_amount = min_stake_amount if stake_amount > max_allowed_stake: - if self._log: + if not self._is_backtest: logger.info( f"Stake amount for pair {pair} is too big " f"({stake_amount} > {max_allowed_stake}), adjusting to {max_allowed_stake}." diff --git a/requirements-dev.txt b/requirements-dev.txt index e0993988a..77d981087 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.2.2 mypy==1.8.0 pre-commit==3.6.2 -pytest==8.0.1 +pytest==8.0.2 pytest-asyncio==0.23.5 pytest-cov==4.1.0 pytest-mock==3.12.0 @@ -21,7 +21,7 @@ isort==5.13.2 time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.16.0 +nbconvert==7.16.1 # mypy types types-cachetools==5.3.0.7 diff --git a/requirements.txt b/requirements.txt index 1c5bf8ba5..0477751cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ numpy==1.26.4 -pandas==2.1.4 +pandas==2.2.1 pandas-ta==0.3.14b -ccxt==4.2.47 -cryptography==42.0.4 +ccxt==4.2.51 +cryptography==42.0.5 aiohttp==3.9.3 SQLAlchemy==2.0.27 python-telegram-bot==20.8 @@ -30,14 +30,14 @@ py_find_1st==1.1.6 # Load ticker files 30% faster python-rapidjson==1.14 # Properly format api responses -orjson==3.9.14 +orjson==3.9.15 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.109.2 -pydantic==2.6.1 +fastapi==0.110.0 +pydantic==2.6.2 uvicorn==0.27.1 pyjwt==2.8.0 aiofiles==23.2.1 @@ -50,6 +50,7 @@ questionary==2.0.1 prompt-toolkit==3.0.36 # Extensions to datetime library python-dateutil==2.8.2 +pytz==2024.1 #Futures schedule==1.2.1 diff --git a/setup.py b/setup.py index 38f0f9a78..dea1966fa 100644 --- a/setup.py +++ b/setup.py @@ -110,6 +110,7 @@ setup( 'cryptography', 'sdnotify', 'python-dateutil', + 'pytz', 'packaging', ], extras_require={ diff --git a/tests/conftest.py b/tests/conftest.py index c1c35fc9d..d894a7908 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -177,7 +177,7 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05', ran def generate_test_data_raw(timeframe: str, size: int, start: str = '2020-07-05', random_seed=42): """ Generates data in the ohlcv format used by ccxt """ df = generate_test_data(timeframe, size, start, random_seed) - df['date'] = df.loc[:, 'date'].view(np.int64) // 1000 // 1000 + df['date'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000 return list(list(x) for x in zip(*(df[x].values.tolist() for x in df.columns))) diff --git a/tests/freqtradebot/test_freqtradebot.py b/tests/freqtradebot/test_freqtradebot.py index aa037fe37..1891c2332 100644 --- a/tests/freqtradebot/test_freqtradebot.py +++ b/tests/freqtradebot/test_freqtradebot.py @@ -4680,9 +4680,14 @@ def test_get_valid_price(mocker, default_conf_usdt) -> None: ('futures', 17, "2021-08-31 23:59:59", "2021-09-01 08:01:07"), ('futures', 17, "2021-08-31 23:59:58", "2021-09-01 08:01:07"), ]) +@pytest.mark.parametrize('tzoffset', [ + '+00:00', + '+01:00', + '-02:00', +]) def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, time_machine, - t1, t2): - time_machine.move_to(f"{t1} +00:00", tick=False) + t1, t2, tzoffset): + time_machine.move_to(f"{t1} {tzoffset}", tick=False) patch_RPCManager(mocker) patch_exchange(mocker) @@ -4691,7 +4696,7 @@ def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, default_conf['margin_mode'] = 'isolated' freqtrade = get_patched_freqtradebot(mocker, default_conf) - time_machine.move_to(f"{t2} +00:00", tick=False) + time_machine.move_to(f"{t2} {tzoffset}", tick=False) # Check schedule jobs in debugging with freqtrade._schedule.jobs freqtrade._schedule.run_pending() diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 7f7bbb29f..2a158acf3 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -57,28 +57,30 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> ), 'close_date': pd.to_datetime([dt_utc(2018, 1, 29, 22, 00, 0), dt_utc(2018, 1, 30, 4, 10, 0)], utc=True), - 'open_rate': [0.10401764894444211, 0.10302485], - 'close_rate': [0.10453904066847439, 0.103541], + 'open_rate': [0.10401764891917063, 0.10302485], + 'close_rate': [0.10453904064307624, 0.10354126528822055], 'fee_open': [0.0025, 0.0025], 'fee_close': [0.0025, 0.0025], 'trade_duration': [200, 40], 'profit_ratio': [0.0, 0.0], 'profit_abs': [0.0, 0.0], 'exit_reason': [ExitType.ROI.value, ExitType.ROI.value], - 'initial_stop_loss_abs': [0.0940005, 0.09272236], + 'initial_stop_loss_abs': [0.0940005, 0.092722365], 'initial_stop_loss_ratio': [-0.1, -0.1], - 'stop_loss_abs': [0.0940005, 0.09272236], + 'stop_loss_abs': [0.0940005, 0.092722365], 'stop_loss_ratio': [-0.1, -0.1], 'min_rate': [0.10370188, 0.10300000000000001], - 'max_rate': [0.10481985, 0.1038888], + 'max_rate': [0.10481985, 0.10388887000000001], 'is_open': [False, False], 'enter_tag': ['', ''], 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], - 'close_timestamp': [1517265300000, 1517285400000], + 'close_timestamp': [1517263200000, 1517285400000], }) - pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) + results_no = results.drop(columns=['orders']) + pd.testing.assert_frame_equal(results_no, expected, check_exact=True) + data_pair = processed[pair] assert len(results.iloc[0]['orders']) == 6 assert len(results.iloc[1]['orders']) == 2 diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 0f190f3f5..e3603849d 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -498,7 +498,7 @@ def test__get_resample_from_period(): assert _get_resample_from_period('day') == '1d' assert _get_resample_from_period('week') == '1W-MON' - assert _get_resample_from_period('month') == '1M' + assert _get_resample_from_period('month') == '1ME' with pytest.raises(ValueError, match=r"Period noooo is not supported."): _get_resample_from_period('noooo') diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 790f5d255..645cae887 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1022,22 +1022,22 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): @pytest.mark.parametrize('function,raises', [ - ('populate_entry_trend', True), + ('populate_entry_trend', False), ('advise_entry', False), - ('populate_exit_trend', True), + ('populate_exit_trend', False), ('advise_exit', False), ]) -def test_pandas_warning_direct(ohlcv_history, function, raises): +def test_pandas_warning_direct(ohlcv_history, function, raises, recwarn): df = _STRATEGY.populate_indicators(ohlcv_history, {'pair': 'ETH/BTC'}) if raises: - with pytest.warns(FutureWarning): - # Test for Future warning - # FutureWarning: Setting an item of incompatible dtype is - # deprecated and will raise in a future error of pandas - # https://github.com/pandas-dev/pandas/issues/56503 - getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + assert len(recwarn) == 1 + # https://github.com/pandas-dev/pandas/issues/56503 + # Fixed in 2.2.x + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) else: + assert len(recwarn) == 0 + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'})