Merge branch 'develop' into feature_keyval_storage

.github/dependabot.yml

@@ -10,8 +10,17 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
+    time: "03:00"
+    timezone: "Etc/UTC"
   open-pull-requests-limit: 15
   target-branch: develop
+  groups:
+    types:
+      patterns:
+        - "types-*"
+    pytest:
+      patterns:
+        - "pytest*"
 
 - package-ecosystem: "github-actions"
   directory: "/"

.github/workflows/ci.yml

@@ -325,7 +325,7 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
 
   docs-check:
     runs-on: ubuntu-22.04

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
           - types-requests==2.31.0.20240125
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.8.19.20240106
-          - SQLAlchemy==2.0.25
+          - SQLAlchemy==2.0.26
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

docs/configuration.md

@@ -14,7 +14,7 @@ You can specify a different configuration file used by the bot with the `-c/--co
 If you used the [Quick start](docker_quickstart.md#docker-quick-start) method for installing
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
 
-If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+If the default configuration file is not created we recommend to use `freqtrade new-config --config user_data/config.json` to generate a basic configuration file.
 
 The Freqtrade configuration file is to be written in JSON format.
 

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.5.7
+mkdocs-material==9.5.9
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.7
 jinja2==3.1.3

docs/utils.md

@@ -54,7 +54,7 @@ optional arguments:
 ### Create config examples
 
 ```
-$ freqtrade new-config --config config_binance.json
+$ freqtrade new-config --config user_data/config_binance.json
 
 ? Do you want to enable Dry-run (simulated trades)?  Yes
 ? Please insert your stake currency: BTC
@@ -990,11 +990,7 @@ options:
   -h, --help            show this help message and exit
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest. Please note that timeframe needs to be set
+                        be converted.
-                        either in config or via command line. When using this
-                        together with `--export trades`, the strategy-name is
-                        injected into the filename (so `backtest-data.json`
-                        becomes `backtest-data-SampleStrategy.json`
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

freqtrade/data/converter/trade_converter_kraken.py

@@ -8,6 +8,7 @@ from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                       trades_df_remove_duplicates)
 from freqtrade.data.history.idatahandler import get_datahandler
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 
 
@@ -38,6 +39,14 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
     }
     logger.info(f"Found csv files for {', '.join(data_symbols)}.")
 
+    if pairs_raw := config.get('pairs'):
+        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
+        markets = {m for m in markets if m[0] in pairs}
+        if not markets:
+            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
+            return
+    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")
+
     for pair, name in markets:
         dfs = []
         # Load and combine all csv files for this pair
@@ -52,17 +61,18 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
             continue
 
         trades = pd.concat(dfs, ignore_index=True)
+        del dfs
 
         trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
         trades.loc[:, 'cost'] = trades['price'] * trades['amount']
         for col in DEFAULT_TRADES_COLUMNS:
             if col not in trades.columns:
-                trades[col] = ''
+                trades.loc[:, col] = ''
-
         trades = trades[DEFAULT_TRADES_COLUMNS]
         trades = trades_convert_types(trades)
 
         trades_df = trades_df_remove_duplicates(trades)
+        del trades
         logger.info(f"{pair}: {len(trades_df)} trades, from "
                     f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                     f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")

freqtrade/rpc/api_server/api_schemas.py

@@ -397,7 +397,7 @@ class ForceEnterPayload(BaseModel):
 
 
 class ForceExitPayload(BaseModel):
-    tradeid: str
+    tradeid: Union[str, int]
     ordertype: Optional[OrderTypeValues] = None
     amount: Optional[float] = None
 

freqtrade/rpc/api_server/api_v1.py

@@ -215,7 +215,7 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
 @router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
 def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)):
     ordertype = payload.ordertype.value if payload.ordertype else None
-    return rpc._rpc_force_exit(payload.tradeid, ordertype, amount=payload.amount)
+    return rpc._rpc_force_exit(str(payload.tradeid), ordertype, amount=payload.amount)
 
 
 @router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])

freqtrade/rpc/telegram.py

@@ -1827,13 +1827,9 @@ class Telegram(RPCHandler):
         msg += f"\nUpdated: {datetime.now().ctime()}"
         if not query.message:
             return
-        chat_id = query.message.chat_id
-        message_id = query.message.message_id
 
         try:
-            await self._app.bot.edit_message_text(
+            await query.edit_message_text(
-                chat_id=chat_id,
-                message_id=message_id,
                 text=msg,
                 parse_mode=parse_mode,
                 reply_markup=reply_markup

requirements-dev.txt

@@ -7,11 +7,11 @@
 -r docs/requirements-docs.txt
 
 coveralls==3.3.1
-ruff==0.2.0
+ruff==0.2.1
 mypy==1.8.0
-pre-commit==3.6.0
+pre-commit==3.6.1
-pytest==7.4.4
+pytest==8.0.0
-pytest-asyncio==0.23.4
+pytest-asyncio==0.23.5
 pytest-cov==4.1.0
 pytest-mock==3.12.0
 pytest-random-order==1.1.1
@@ -21,7 +21,7 @@ isort==5.13.2
 time-machine==2.13.0
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.14.2
+nbconvert==7.16.0
 
 # mypy types
 types-cachetools==5.3.0.7

requirements-freqai-rl.txt

@@ -8,4 +8,4 @@ gymnasium==0.29.1; python_version < '3.12'
 stable_baselines3==2.2.1; python_version < '3.12'
 sb3_contrib>=2.0.0a9; python_version < '3.12'
 # Progress bar for stable-baselines3 and sb3-contrib
-tqdm==4.66.1
+tqdm==4.66.2

requirements-freqai.txt

@@ -8,5 +8,5 @@ joblib==1.3.2
 catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12'
 lightgbm==4.3.0
 xgboost==2.0.3
-tensorboard==2.15.1
+tensorboard==2.15.2
 datasieve==0.1.7

requirements.txt

@@ -1,12 +1,12 @@
-numpy==1.26.3
+numpy==1.26.4
 pandas==2.1.4
 pandas-ta==0.3.14b
 
-ccxt==4.2.35
+ccxt==4.2.42
 cryptography==42.0.2
 aiohttp==3.9.3
-SQLAlchemy==2.0.25
+SQLAlchemy==2.0.26
-python-telegram-bot==20.7
+python-telegram-bot==20.8
 # can't be hard-pinned due to telegram-bot pinning httpx with ~
 httpx>=0.24.1
 arrow==1.3.0
@@ -38,7 +38,7 @@ sdnotify==0.3.2
 # API Server
 fastapi==0.109.2
 pydantic==2.6.1
-uvicorn==0.27.0.post1
+uvicorn==0.27.1
 pyjwt==2.8.0
 aiofiles==23.2.1
 psutil==5.9.8

tests/data/test_trade_converter_kraken.py

@@ -34,6 +34,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
 
     import_kraken_trades_from_csv(default_conf_usdt, 'feather')
     assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("Converting pairs: BCH/EUR.", caplog)
     assert log_has_re(r"BCH/EUR: 340 trades.* 2023-01-01.* 2023-01-02.*", caplog)
 
     assert dstfile.is_file()
@@ -48,3 +49,10 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
                                                             tzinfo=timezone.utc)
     # ID is not filled
     assert len(trades.loc[trades['id'] != '']) == 0
+
+    caplog.clear()
+    default_conf_usdt['pairs'] = ['XRP/EUR']
+    # Filtered to non-existing pair
+    import_kraken_trades_from_csv(default_conf_usdt, 'feather')
+    assert log_has("Found csv files for BCHEUR.", caplog)
+    assert log_has("No data found for pairs XRP/EUR.", caplog)

tests/rpc/test_rpc_telegram.py

@@ -2557,22 +2557,22 @@ async def test_telegram__send_msg(default_conf, mocker, caplog) -> None:
 
     # Test update
     query = MagicMock()
+    query.edit_message_text = AsyncMock()
     await telegram._send_msg('test', callback_path="DeadBeef", query=query, reload_able=True)
-    edit_message_text = telegram._app.bot.edit_message_text
+    assert query.edit_message_text.call_count == 1
-    assert edit_message_text.call_count == 1
+    assert "Updated: " in query.edit_message_text.call_args_list[0][1]['text']
-    assert "Updated: " in edit_message_text.call_args_list[0][1]['text']
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=BadRequest("not modified"))
+    query.edit_message_text = AsyncMock(side_effect=BadRequest("not modified"))
     await telegram._send_msg('test', callback_path="DeadBeef", query=query)
-    assert telegram._app.bot.edit_message_text.call_count == 1
+    assert query.edit_message_text.call_count == 1
     assert not log_has_re(r"TelegramError: .*", caplog)
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=BadRequest(""))
+    query.edit_message_text = AsyncMock(side_effect=BadRequest(""))
     await telegram._send_msg('test2', callback_path="DeadBeef", query=query)
-    assert telegram._app.bot.edit_message_text.call_count == 1
+    assert query.edit_message_text.call_count == 1
     assert log_has_re(r"TelegramError: .*", caplog)
 
-    telegram._app.bot.edit_message_text = AsyncMock(side_effect=TelegramError("DeadBEEF"))
+    query.edit_message_text = AsyncMock(side_effect=TelegramError("DeadBEEF"))
     await telegram._send_msg('test3', callback_path="DeadBeef", query=query)
 
     assert log_has_re(r"TelegramError: DeadBEEF! Giving up.*", caplog)