Merge pull request #10464 from freqtrade/new_release

New release 2024.7

.devcontainer/devcontainer.json

@@ -18,15 +18,22 @@
                 "editor.insertSpaces": true,
                 "files.trimTrailingWhitespace": true,
                 "[markdown]": {
-                    "files.trimTrailingWhitespace": false,
+                    "files.trimTrailingWhitespace": false
                 },
                 "python.pythonPath": "/usr/local/bin/python",
+                "[python]": {
+                    "editor.codeActionsOnSave": {
+                        "source.organizeImports": "explicit"
+                    },
+                    "editor.formatOnSave": true,
+                    "editor.defaultFormatter": "charliermarsh.ruff"
+                }
             },
             // Add the IDs of extensions you want installed when the container is created.
             "extensions": [
                 "ms-python.python",
                 "ms-python.vscode-pylance",
-                "ms-python.isort",
+                "charliermarsh.ruff",
                 "davidanson.vscode-markdownlint",
                 "ms-azuretools.vscode-docker",
                 "vscode-icons-team.vscode-icons",

.github/workflows/binance-lev-tier-update.yml

@@ -19,7 +19,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Install ccxt
       run: pip install ccxt

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, ubuntu-22.04 ]
+        os: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -55,7 +55,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -80,6 +80,11 @@ jobs:
         # Allow failure for coveralls
         coveralls || true
 
+    - name: Run json schema extract
+      # This should be kept before the repository check to ensure that the schema is up-to-date
+      run: |
+        python build_helpers/extract_config_json_schema.py
+
     - name: Check for repository changes
       run: |
         if [ -n "$(git status --porcelain)" ]; then
@@ -111,7 +116,11 @@ jobs:
 
     - name: Run Ruff
       run: |
-        ruff check --output-format=github .
+        ruff check --output-format=github
+
+    - name: Run Ruff format check
+      run: |
+        ruff format --check
 
     - name: Mypy
       run: |
@@ -188,7 +197,7 @@ jobs:
 
     - name: Installation (python)
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -230,7 +239,11 @@ jobs:
 
     - name: Run Ruff
       run: |
-        ruff check --output-format=github .
+        ruff check --output-format=github
+
+    - name: Run Ruff format check
+      run: |
+        ruff format --check
 
     - name: Mypy
       run: |
@@ -300,12 +313,27 @@ jobs:
 
     - name: Run Ruff
       run: |
-        ruff check --output-format=github .
+        ruff check --output-format=github
+
+    - name: Run Ruff format check
+      run: |
+        ruff format --check
 
     - name: Mypy
       run: |
         mypy freqtrade scripts tests
 
+    - name: Run Pester tests (PowerShell)
+      run: |
+        $PSVersionTable
+        Set-PSRepository psgallery -InstallationPolicy trusted
+        Install-Module -Name Pester -RequiredVersion 5.3.1 -Confirm:$false -Force -SkipPublisherCheck
+        $Error.clear()
+        Invoke-Pester -Path "tests" -CI
+        if ($Error.Length -gt 0) {exit 1}
+
+      shell: powershell
+
     - name: Discord notification
       uses: rjstone/discord-webhook-notify@v1
       if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
@@ -322,7 +350,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: pre-commit dependencies
       run: |
@@ -336,7 +364,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
     - uses: pre-commit/action@v3.0.1
 
   docs-check:
@@ -351,7 +379,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Documentation build
       run: |
@@ -377,7 +405,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Cache_dependencies
       uses: actions/cache@v4
@@ -399,7 +427,7 @@ jobs:
 
     - name: Installation - *nix
       run: |
-        python -m pip install --upgrade pip wheel
+        python -m pip install --upgrade "pip<=24.0" wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
         export TA_INCLUDE_PATH=${HOME}/dependencies/include
@@ -459,7 +487,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Build distribution
       run: |
@@ -510,12 +538,12 @@ jobs:
 
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0
       with:
         repository-url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0
 
 
   deploy-docker:
@@ -530,7 +558,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Extract branch name
       id: extract-branch
@@ -553,12 +581,12 @@ jobs:
           sudo systemctl restart docker
           docker version -f '{{.Server.Experimental}}'
 
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
       id: buildx
-      uses: crazy-max/ghaction-docker-buildx@v3.3.1
-      with:
-        buildx-version: latest
-        qemu-version: latest
+      uses: docker/setup-buildx-action@v3
 
     - name: Available platforms
       run: echo ${{ steps.buildx.outputs.platforms }}

.github/workflows/devcontainer-build.yml

@@ -2,6 +2,8 @@ name: Devcontainer Pre-Build
 
 on:
   workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * 0"
   # push:
   #   branches:
   #     - "master"

.github/workflows/pre-commit-update.yml

@@ -17,7 +17,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
 
     - name: Install pre-commit
@@ -26,9 +26,6 @@ jobs:
     - name: Run auto-update
       run: pre-commit autoupdate
 
-    - name: Run pre-commit
-      run: pre-commit run --all-files
-
     - uses: peter-evans/create-pull-request@v6
       with:
         token: ${{ secrets.REPO_SCOPED_TOKEN }}

.pre-commit-config.yaml

@@ -2,24 +2,24 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pycqa/flake8
-    rev: "7.0.0"
+    rev: "7.1.0"
     hooks:
       - id: flake8
         additional_dependencies: [Flake8-pyproject]
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.11.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
-          - types-cachetools==5.3.0.7
+          - types-cachetools==5.4.0.20240717
           - types-filelock==3.2.7
-          - types-requests==2.31.0.20240406
+          - types-requests==2.32.0.20240712
           - types-tabulate==0.9.0.20240106
           - types-python-dateutil==2.9.0.20240316
-          - SQLAlchemy==2.0.29
+          - SQLAlchemy==2.0.31
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.2'
+    rev: 'v0.5.4'
     hooks:
       - id: ruff
 
@@ -56,7 +56,7 @@ repos:
             )$
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
     - id: codespell
       additional_dependencies:

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+    "recommendations": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "charliermarsh.ruff",
+        "davidanson.vscode-markdownlint",
+        "ms-azuretools.vscode-docker",
+        "vscode-icons-team.vscode-icons",
+        "github.vscode-github-actions",
+    ]
+}

CONTRIBUTING.md

@@ -72,12 +72,12 @@ you can manually run pre-commit with `pre-commit run -a`.
 mypy freqtrade
 ```
 
-### 4. Ensure all imports are correct
+### 4. Ensure formatting is correct
 
-#### Run isort
+#### Run ruff
 
 ``` bash
-isort .
+ruff format .
 ```
 
 ## (Core)-Committer Guide

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12.3-slim-bookworm as base
+FROM python:3.12.4-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8
@@ -25,7 +25,7 @@ FROM base as python-deps
 RUN  apt-get update \
   && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
-  && pip install --upgrade pip wheel
+  && pip install --upgrade "pip<=24.0" wheel
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
@@ -35,7 +35,7 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir "numpy<2.0" \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
 # Copy dependencies to runtime-image

README.md

@@ -29,6 +29,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
+- [X] [BingX](https://bingx.com/invite/0EM9RX)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)

build_helpers/binance_update_lev_tiers.py

@@ -6,21 +6,18 @@ from pathlib import Path
 import ccxt
 
 
-key = os.environ.get('FREQTRADE__EXCHANGE__KEY')
-secret = os.environ.get('FREQTRADE__EXCHANGE__SECRET')
+key = os.environ.get("FREQTRADE__EXCHANGE__KEY")
+secret = os.environ.get("FREQTRADE__EXCHANGE__SECRET")
 
-proxy = os.environ.get('CI_WEB_PROXY')
+proxy = os.environ.get("CI_WEB_PROXY")
 
-exchange = ccxt.binance({
-    'apiKey': key,
-    'secret': secret,
-    'httpsProxy': proxy,
-    'options': {'defaultType': 'swap'}
-    })
+exchange = ccxt.binance(
+    {"apiKey": key, "secret": secret, "httpsProxy": proxy, "options": {"defaultType": "swap"}}
+)
 _ = exchange.load_markets()
 
 lev_tiers = exchange.fetch_leverage_tiers()
 
 # Assumes this is running in the root of the repository.
-file = Path('freqtrade/exchange/binance_leverage_tiers.json')
-json.dump(dict(sorted(lev_tiers.items())), file.open('w'), indent=2)
+file = Path("freqtrade/exchange/binance_leverage_tiers.json")
+json.dump(dict(sorted(lev_tiers.items())), file.open("w"), indent=2)

build_helpers/extract_config_json_schema.py

@@ -0,0 +1,17 @@
+"""Script to extract the configuration json schema from config_schema.py file."""
+
+from pathlib import Path
+
+import rapidjson
+
+from freqtrade.configuration.config_schema import CONF_SCHEMA
+
+
+def extract_config_json_schema():
+    schema_filename = Path(__file__).parent / "schema.json"
+    with schema_filename.open("w") as f:
+        rapidjson.dump(CONF_SCHEMA, f, indent=2)
+
+
+if __name__ == "__main__":
+    extract_config_json_schema()

build_helpers/freqtrade_client_version_align.py

@@ -1,18 +1,15 @@
 #!/usr/bin/env python3
-from freqtrade_client import __version__ as client_version
-
 from freqtrade import __version__ as ft_version
+from freqtrade_client import __version__ as client_version
 
 
 def main():
     if ft_version != client_version:
-        print(f"Versions do not match: \n"
-              f"ft: {ft_version} \n"
-              f"client: {client_version}")
+        print(f"Versions do not match: \nft: {ft_version} \nclient: {client_version}")
         exit(1)
     print(f"Versions match: ft: {ft_version}, client: {client_version}")
     exit(0)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

build_helpers/install_windows.ps1

@@ -1,6 +1,6 @@
 # vendored Wheels compiled via https://github.com/xmatthias/ta-lib-python/tree/ta_bundled_040
 
-python -m pip install --upgrade pip wheel
+python -m pip install --upgrade "pip<=24.0" wheel
 
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 

build_helpers/pre_commit_update.py

@@ -6,28 +6,30 @@ from pathlib import Path
 import yaml
 
 
-pre_commit_file = Path('.pre-commit-config.yaml')
-require_dev = Path('requirements-dev.txt')
-require = Path('requirements.txt')
+pre_commit_file = Path(".pre-commit-config.yaml")
+require_dev = Path("requirements-dev.txt")
+require = Path("requirements.txt")
 
-with require_dev.open('r') as rfile:
+with require_dev.open("r") as rfile:
     requirements = rfile.readlines()
 
-with require.open('r') as rfile:
+with require.open("r") as rfile:
     requirements.extend(rfile.readlines())
 
 # Extract types only
-type_reqs = [r.strip('\n') for r in requirements if r.startswith(
-    'types-') or r.startswith('SQLAlchemy')]
+type_reqs = [
+    r.strip("\n") for r in requirements if r.startswith("types-") or r.startswith("SQLAlchemy")
+]
 
-with pre_commit_file.open('r') as file:
+with pre_commit_file.open("r") as file:
     f = yaml.load(file, Loader=yaml.SafeLoader)
 
 
-mypy_repo = [repo for repo in f['repos'] if repo['repo']
-             == 'https://github.com/pre-commit/mirrors-mypy']
+mypy_repo = [
+    repo for repo in f["repos"] if repo["repo"] == "https://github.com/pre-commit/mirrors-mypy"
+]
 
-hooks = mypy_repo[0]['hooks'][0]['additional_dependencies']
+hooks = mypy_repo[0]["hooks"][0]["additional_dependencies"]
 
 errors = []
 for hook in hooks:

docker-compose.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable

docker/Dockerfile.armhf

@@ -17,7 +17,7 @@ RUN mkdir /freqtrade \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
   && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
-  && pip install --upgrade pip
+  && pip install --upgrade "pip<=24.0"
 
 WORKDIR /freqtrade
 
@@ -35,7 +35,7 @@ COPY build_helpers/* /tmp/
 COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
 USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
-  && pip install --user --no-index --find-links /tmp/ pyarrow TA-Lib==0.4.28 \
+  && pip install --user --no-index --find-links /tmp/ pyarrow TA-Lib \
   && pip install --user --no-cache-dir -r requirements.txt
 
 # Copy dependencies to runtime-image

docker/docker-compose-freqai.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   freqtrade:
     image: freqtradeorg/freqtrade:stable_freqaitorch

docker/docker-compose-jupyter.yml

@@ -1,5 +1,4 @@
 ---
-version: '3'
 services:
   ft_jupyterlab:
     build:

docs/advanced-orderflow.md

@@ -0,0 +1,152 @@
+# Orderflow data
+
+This guide walks you through utilizing public trade data for advanced orderflow analysis in Freqtrade.
+
+!!! Warning "Experimental Feature"
+    The orderflow feature is currently in beta and may be subject to changes in future releases. Please report any issues or feedback on the [Freqtrade GitHub repository](https://github.com/freqtrade/freqtrade/issues).
+
+!!! Warning "Performance"
+    Orderflow requires raw trades data. This data is rather large, and can cause a slow initial startup, when freqtrade needs to download the trades data for the last X candles. Additionally, enabling this feature will cause increased memory usage. Please ensure to have sufficient resources available.
+
+## Getting Started
+
+### Enable Public Trades
+
+In your `config.json` file, set the `use_public_trades` option to true under the `exchange` section.
+
+```json
+"exchange": {
+   ...
+   "use_public_trades": true,
+}
+```
+
+### Configure Orderflow Processing
+
+Define your desired settings for orderflow processing within the orderflow section of config.json. Here, you can adjust factors like:
+
+- `cache_size`: How many previous orderflow candles are saved into cache instead of calculated every new candle
+- `max_candles`: Filter how many candles would you like to get trades data for.
+- `scale`: This controls the price bin size for the footprint chart.
+- `stacked_imbalance_range`: Defines the minimum consecutive imbalanced price levels required for consideration.
+- `imbalance_volume`: Filters out imbalances with volume below this threshold.
+- `imbalance_ratio`: Filters out imbalances with a ratio (difference between ask and bid volume) lower than this value.
+
+```json
+"orderflow": {
+    "cache_size": 1000, 
+    "max_candles": 1500, 
+    "scale": 0.5, 
+    "stacked_imbalance_range": 3, //  needs at least this amount of imbalance next to each other
+    "imbalance_volume": 1, //  filters out below
+    "imbalance_ratio": 3 //  filters out ratio lower than
+  },
+```
+
+## Downloading Trade Data for Backtesting
+
+To download historical trade data for backtesting, use the --dl-trades flag with the freqtrade download-data command.
+
+```bash
+freqtrade download-data -p BTC/USDT:USDT --timerange 20230101- --trading-mode futures --timeframes 5m --dl-trades
+```
+
+!!! Warning "Data availability"
+    Not all exchanges provide public trade data. For supported exchanges, freqtrade will warn you if public trade data is not available if you start downloading data with the `--dl-trades` flag.
+
+## Accessing Orderflow Data
+
+Once activated, several new columns become available in your dataframe:
+
+``` python
+
+dataframe["trades"] # Contains information about each individual trade.
+dataframe["orderflow"] # Represents a footprint chart dict (see below)
+dataframe["imbalances"] # Contains information about imbalances in the order flow.
+dataframe["bid"] # Total bid volume 
+dataframe["ask"] # Total ask volume
+dataframe["delta"] # Difference between ask and bid volume.
+dataframe["min_delta"] # Minimum delta within the candle
+dataframe["max_delta"] # Maximum delta within the candle
+dataframe["total_trades"] # Total number of trades
+dataframe["stacked_imbalances_bid"] # Price level of stacked bid imbalance 
+dataframe["stacked_imbalances_ask"] # Price level of stacked ask imbalance  
+```
+
+You can access these columns in your strategy code for further analysis. Here's an example:
+
+``` python
+def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    # Calculating cumulative delta
+    dataframe["cum_delta"] = cumulative_delta(dataframe["delta"])
+    # Accessing total trades
+    total_trades = dataframe["total_trades"]
+    ...
+
+def cumulative_delta(delta: Series):
+    cumdelta = delta.cumsum()
+    return cumdelta
+
+```
+
+### Footprint chart (`dataframe["orderflow"]`)
+
+This column provides a detailed breakdown of buy and sell orders at different price levels, offering valuable insights into order flow dynamics. The `scale` parameter in your configuration determines the price bin size for this representation
+
+The `orderflow` column contains a dict with the following structure:
+
+``` output
+{
+    "price": {
+        "bid_amount": 0.0,
+        "ask_amount": 0.0,
+        "bid": 0,
+        "ask": 0,
+        "delta": 0.0,
+        "total_volume": 0.0,
+        "total_trades": 0
+    }
+}
+```
+
+#### Orderflow column explanation
+
+- key: Price bin - binned at `scale` intervals
+- `bid_amount`: Total volume bought at each price level.
+- `ask_amount`: Total volume sold at each price level.
+- `bid`: Number of buy orders at each price level.
+- `ask`: Number of sell orders at each price level.
+- `delta`: Difference between ask and bid volume at each price level.
+- `total_volume`: Total volume (ask amount + bid amount) at each price level.
+- `total_trades`: Total number of trades (ask + bid) at each price level.
+
+By leveraging these features, you can gain valuable insights into market sentiment and potential trading opportunities based on order flow analysis.
+
+### Raw trades data (`dataframe["trades"]`)
+
+List with the individual trades that occurred during the candle. This data can be used for more granular analysis of order flow dynamics.
+
+Each individual entry contains a dict with the following keys:
+
+- `timestamp`: Timestamp of the trade.
+- `date`: Date of the trade.
+- `price`: Price of the trade.
+- `amount`: Volume of the trade.
+- `side`: Buy or sell.
+- `id`: Unique identifier for the trade.
+- `cost`: Total cost of the trade (price * amount).
+
+### Imbalances (`dataframe["imbalances"]`)
+
+This column provides a dict with information about imbalances in the order flow. An imbalance occurs when there is a significant difference between the ask and bid volume at a given price level.
+
+Each row looks as follows - with price as index, and the corresponding bid and ask imbalance values as columns
+
+``` output
+{
+    "price": {
+        "bid_imbalance": False,
+        "ask_imbalance": False
+    }
+}
+```

docs/advanced-setup.md

@@ -114,8 +114,46 @@ services:
       --strategy SampleStrategy
 
 ```
+
 You can use whatever naming convention you want, freqtrade1 and 2 are arbitrary. Note, that you will need to use different database files, port mappings and telegram configurations for each instance, as mentioned above. 
 
+## Use a different database system
+
+Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
+Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
+
+The following systems have been tested and are known to work with freqtrade:
+
+* sqlite (default)
+* PostgreSQL
+* MariaDB
+
+!!! Warning
+    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
+
+### PostgreSQL
+
+Installation:
+`pip install psycopg2-binary`
+
+Usage:
+`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
+
+Freqtrade will automatically create the tables necessary upon startup.
+
+If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
+
+### MariaDB / MySQL
+
+Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
+
+Installation:
+`pip install pymysql`
+
+Usage:
+`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`
+
+
 
 ## Configure the bot running as a systemd service
 

docs/backtesting.md

@@ -253,36 +253,36 @@ A backtesting result will look like that:
 
 ```
 ================================================ BACKTESTING REPORT =================================================
-| Pair     | Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
-|:---------|--------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
-| ADA/BTC  |      35 |          -0.11 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
-| ARK/BTC  |      11 |          -0.41 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
-| BTS/BTC  |      32 |           0.31 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
-| DASH/BTC |      13 |          -0.08 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
-| ENG/BTC  |      18 |           1.36 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
-| EOS/BTC  |      36 |           0.08 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
-| ETC/BTC  |      26 |           0.37 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
-| ETH/BTC  |      33 |           0.30 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
-| IOTA/BTC |      32 |           0.03 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
-| LSK/BTC  |      15 |           1.75 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
-| LTC/BTC  |      32 |          -0.04 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
-| NANO/BTC |      17 |           1.26 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
-| NEO/BTC  |      23 |           0.82 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
-| REQ/BTC  |       9 |           1.17 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
-| XLM/BTC  |      16 |           1.22 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
-| XMR/BTC  |      23 |          -0.18 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
-| XRP/BTC  |      35 |           0.66 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
-| ZEC/BTC  |      22 |          -0.46 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
-| TOTAL    |     429 |           0.36 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
+| Pair     | Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration |  Wins Draws Loss   Win%  |
+|----------+--------+----------------+------------------+----------------+--------------+--------------------------|
+| ADA/BTC  |     35 |          -0.11 |      -0.00019428 |          -1.94 | 4:35:00      |    14     0    21   40.0 |
+| ARK/BTC  |     11 |          -0.41 |      -0.00022647 |          -2.26 | 2:03:00      |     3     0     8   27.3 |
+| BTS/BTC  |     32 |           0.31 |       0.00048938 |           4.89 | 5:05:00      |    18     0    14   56.2 |
+| DASH/BTC |     13 |          -0.08 |      -0.00005343 |          -0.53 | 4:39:00      |     6     0     7   46.2 |
+| ENG/BTC  |     18 |           1.36 |       0.00122807 |          12.27 | 2:50:00      |     8     0    10   44.4 |
+| EOS/BTC  |     36 |           0.08 |       0.00015304 |           1.53 | 3:34:00      |    16     0    20   44.4 |
+| ETC/BTC  |     26 |           0.37 |       0.00047576 |           4.75 | 6:14:00      |    11     0    15   42.3 |
+| ETH/BTC  |     33 |           0.30 |       0.00049856 |           4.98 | 7:31:00      |    16     0    17   48.5 |
+| IOTA/BTC |     32 |           0.03 |       0.00005444 |           0.54 | 3:12:00      |    14     0    18   43.8 |
+| LSK/BTC  |     15 |           1.75 |       0.00131413 |          13.13 | 2:58:00      |     6     0     9   40.0 |
+| LTC/BTC  |     32 |          -0.04 |      -0.00006886 |          -0.69 | 4:49:00      |    11     0    21   34.4 |
+| NANO/BTC |     17 |           1.26 |       0.00107058 |          10.70 | 1:55:00      |    10     0     7   58.5 |
+| NEO/BTC  |     23 |           0.82 |       0.00094936 |           9.48 | 2:59:00      |    10     0    13   43.5 |
+| REQ/BTC  |      9 |           1.17 |       0.00052734 |           5.27 | 3:47:00      |     4     0     5   44.4 |
+| XLM/BTC  |     16 |           1.22 |       0.00097800 |           9.77 | 3:15:00      |     7     0     9   43.8 |
+| XMR/BTC  |     23 |          -0.18 |      -0.00020696 |          -2.07 | 5:30:00      |    12     0    11   52.2 |
+| XRP/BTC  |     35 |           0.66 |       0.00114897 |          11.48 | 3:49:00      |    12     0    23   34.3 |
+| ZEC/BTC  |     22 |          -0.46 |      -0.00050971 |          -5.09 | 2:22:00      |     7     0    15   31.8 |
+| TOTAL    |    429 |           0.36 |       0.00762792 |          76.20 | 4:12:00      |   186     0   243   43.4 |
 ============================================= LEFT OPEN TRADES REPORT =============================================
-| Pair     |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
-|:---------|---------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
-| ADA/BTC  |        1 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
-| LTC/BTC  |        1 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
-| TOTAL    |        2 |           0.78 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
+| Pair     |  Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Win Draw Loss Win% |
+|----------+---------+----------------+------------------+----------------+----------------+---------------------|
+| ADA/BTC  |       1 |           0.89 |       0.00004434 |           0.44 | 6:00:00        |    1    0    0  100 |
+| LTC/BTC  |       1 |           0.68 |       0.00003421 |           0.34 | 2:00:00        |    1    0    0  100 |
+| TOTAL    |       2 |           0.78 |       0.00007855 |           0.78 | 4:00:00        |    2    0    0  100 |
 ==================== EXIT REASON STATS ====================
 | Exit Reason        |   Exits |  Wins |  Draws |  Losses |
-|:-------------------|--------:|------:|-------:|--------:|
+|--------------------+---------+-------+--------+---------|
 | trailing_stop_loss |     205 |   150 |      0 |      55 |
 | stop_loss          |     166 |     0 |      0 |     166 |
 | exit_signal        |      56 |    36 |      0 |      20 |
@@ -631,10 +631,10 @@ Detailed output for all strategies one after the other will be available, so mak
 
 ```
 ================================================== STRATEGY SUMMARY ===================================================================
-| Strategy    |  Entries |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
-|:------------|---------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
-| Strategy1   |      429 |           0.36 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
-| Strategy2   |     1487 |          -0.13 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
+| Strategy    |  Trades |   Avg Profit % |   Tot Profit BTC |   Tot Profit % | Avg Duration   |  Wins |  Draws | Losses | Drawdown % |
+|-------------+---------+----------------+------------------+----------------+----------------+-------+--------+--------+------------|
+| Strategy1   |     429 |           0.36 |       0.00762792 |          76.20 | 4:12:00        |   186 |      0 |    243 |       45.2 |
+| Strategy2   |    1487 |          -0.13 |      -0.00988917 |         -98.79 | 4:43:00        |   662 |      0 |    825 |     241.68 |
 ```
 
 ## Next step

docs/configuration.md

@@ -204,9 +204,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
 | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
-| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
-| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
-| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
+| `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://docs.ccxt.com/#/README?id=overriding-exchange-properties-upon-instantiation) <br> **Datatype:** Dict
+| `exchange.enable_ws` | Enable the usage of Websockets for the exchange. <br>[More information](#consuming-exchange-websockets).<br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer
 | `exchange.skip_pair_validation` | Skip pairlist validation on startup.<br>*Defaults to `false`*<br> **Datatype:** Boolean
 | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.<br>*Defaults to `false`*<br> **Datatype:** Boolean
@@ -409,6 +410,8 @@ Or another example if your position adjustment assumes it can do 1 additional bu
 
 --8<-- "includes/pricing.md"
 
+## Further Configuration details
+
 ### Understand minimal_roi
 
 The `minimal_roi` configuration parameter is a JSON object where the key is a duration
@@ -568,7 +571,14 @@ The possible values are: `GTC` (default), `FOK` or `IOC`.
     This is ongoing work. For now, it is supported only for binance, gate and kucoin.
     Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange.
 
-### What values can be used for fiat_display_currency?
+### Fiat conversion
+
+Freqtrade uses the Coingecko API to convert the coin value to it's corresponding fiat value for the Telegram reports.
+The FIAT currency can be set in the configuration file as `fiat_display_currency`.
+
+Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot.
+
+#### What values can be used for fiat_display_currency?
 
 The `fiat_display_currency` configuration parameter sets the base currency to use for the
 conversion from coin to fiat in the bot Telegram reports.
@@ -587,7 +597,49 @@ The valid values are:
 "BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ```
 
-Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot.
+#### Coingecko Rate limit problems
+
+On some IP ranges, coingecko is heavily rate-limiting.
+In such cases, you may want to add your coingecko API key to the configuration.
+
+``` json
+{
+    "fiat_display_currency": "USD",
+    "coingecko": {
+        "api_key": "your-api",
+        "is_demo": true
+    }
+}
+```
+
+Freqtrade supports both Demo and Pro coingecko API keys.
+
+The Coingecko API key is NOT required for the bot to function correctly.
+It is only used for the conversion of coin to fiat in the Telegram reports, which usually also work without API key.
+
+## Consuming exchange Websockets
+
+Freqtrade can consume websockets through ccxt.pro.
+
+Freqtrade aims ensure data is available at all times.
+Should the websocket connection fail (or be disabled), the bot will fall back to REST API calls.
+
+Should you experience problems you suspect are caused by websockets, you can disable these via the setting `exchange.enable_ws`, which defaults to true.
+
+```jsonc
+"exchange": {
+    // ...
+    "enable_ws": false,
+    // ...
+}
+```
+
+Should you be required to use a proxy, please refer to the [proxy section](#using-proxy-with-freqtrade) for more information.
+
+!!! Info "Rollout"
+    We're implementing this out slowly, ensuring stability of your bots.
+    Currently, usage is limited to ohlcv data streams.
+    It's also limited to a few exchanges, with new exchanges being added on an ongoing basis.
 
 ## Using Dry-run mode
 
@@ -625,9 +677,9 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode.
 * Wallets (`/balance`) are simulated based on `dry_run_wallet`.
 * Orders are simulated, and will not be posted to the exchange.
-* Market orders fill based on orderbook volume the moment the order is placed.
+* Market orders fill based on orderbook volume the moment the order is placed, with a maximum slippage of 5%.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
-* Limit orders will be converted to market orders if they cross the price by more than 1%.
+* Limit orders will be converted to market orders if they cross the price by more than 1%, and will be filled immediately based regular market order rules (see point about Market orders above).
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 
@@ -677,7 +729,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 
     **NEVER** share your private configuration file or your exchange keys with anyone!
 
-### Using proxy with Freqtrade
+## Using a proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
 This will have the proxy settings applied to everything (telegram, coingecko, ...) **except** for exchange requests.
@@ -688,7 +740,7 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy exchange requests
+### Proxy exchange requests
 
 To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
@@ -697,6 +749,7 @@ To use a proxy for exchange connections - you will have to define the proxies as
   "exchange": {
     "ccxt_config": {
       "httpsProxy": "http://addr:port",
+      "wsProxy": "http://addr:port",
     }
   }
 }

docs/data-download.md

@@ -24,10 +24,10 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--days INT] [--new-pairs-days INT]
                                [--include-inactive-pairs]
                                [--timerange TIMERANGE] [--dl-trades]
-                               [--exchange EXCHANGE]
+                               [--convert] [--exchange EXCHANGE]
                                [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
-                               [--data-format-trades {json,jsongz,hdf5,feather}]
+                               [--data-format-trades {json,jsongz,hdf5,feather,parquet}]
                                [--trading-mode {spot,margin,futures}]
                                [--prepend]
 
@@ -48,6 +48,11 @@ options:
   --dl-trades           Download trades instead of OHLCV data. The bot will
                         resample trades to the desired timeframe as specified
                         as --timeframes/-t.
+  --convert             Convert downloaded trades to OHLCV data. Only
+                        applicable in combination with `--dl-trades`. Will be
+                        automatic for exchanges which don't have historic
+                        OHLCV (e.g. Kraken). If not provided, use `trades-to-
+                        ohlcv` to convert trades data to OHLCV data.
   --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
@@ -57,7 +62,7 @@ options:
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
                         (default: `feather`).
-  --data-format-trades {json,jsongz,hdf5,feather}
+  --data-format-trades {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded trades data. (default:
                         `feather`).
   --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
@@ -471,15 +476,20 @@ ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
 
 ## Trades (tick) data
 
-By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+By default, `download-data` sub-command downloads Candles (OHLCV) data. Most exchanges also provide historic trade-data via their API.
 This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
 
-Since this data is large by default, the files use the feather fileformat by default. They are stored in your data-directory with the naming convention of `<pair>-trades.feather` (`ETH_BTC-trades.feather`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+Since this data is large by default, the files use the feather file format by default. They are stored in your data-directory with the naming convention of `<pair>-trades.feather` (`ETH_BTC-trades.feather`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
 
-To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
+To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades.
+If `--convert` is also provided, the resample step will happen automatically and overwrite eventually existing OHLCV data for the given pair/timeframe combinations.
 
-!!! Warning "do not use"
-    You should not use this unless you're a kraken user. Most other exchanges provide OHLCV data with sufficient history.
+!!! Warning "Do not use"
+    You should not use this unless you're a kraken user (Kraken does not provide historic OHLCV data).  
+    Most other exchanges provide OHLCV data with sufficient history, so downloading multiple timeframes through that method will still proof to be a lot faster than downloading trades data.
+
+!!! Note "Kraken user"
+    Kraken users should read [this](exchanges.md#historic-kraken-data) before starting to download data.
 
 Example call:
 
@@ -490,12 +500,6 @@ freqtrade download-data --exchange kraken --pairs XRP/EUR ETH/EUR --days 20 --dl
 !!! Note
     While this method uses async calls, it will be slow, since it requires the result of the previous call to generate the next request to the exchange.
 
-!!! Warning
-    The historic trades are not available during Freqtrade dry-run and live trade modes because all exchanges tested provide this data with a delay of few 100 candles, so it's not suitable for real-time trading.
-
-!!! Note "Kraken user"
-    Kraken users should read [this](exchanges.md#historic-kraken-data) before starting to download data.
-
 ## Next step
 
-Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy.
+Great, you now have some data downloaded, so you can now start [backtesting](backtesting.md) your strategy.

docs/developer.md

@@ -22,7 +22,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt
 ## Developer setup
 
 To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
-Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
+Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -r requirements-dev.txt` - followed by `pip3 install -e .[all]`.
 
 This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`.
 

docs/exchanges.md

@@ -127,6 +127,13 @@ These settings will be checked on startup, and freqtrade will show an error if t
 
 Freqtrade will not attempt to change these settings.
 
+## Bingx
+
+BingX supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings.
+
+!!! Tip "Stoploss on Exchange"
+    Bingx supports `stoploss_on_exchange` and can use both stop-limit and stop-market orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
+
 ## Kraken
 
 Kraken supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings.

docs/freq-ui.md

@@ -0,0 +1,85 @@
+# FreqUI
+
+Freqtrade provides a builtin webserver, which can serve [FreqUI](https://github.com/freqtrade/frequi), the freqtrade frontend.
+
+By default, the UI is automatically installed as part of the installation (script, docker).
+freqUI can also be manually installed by using the `freqtrade install-ui` command.
+This same command can also be used to update freqUI to new new releases.
+
+Once the bot is started in trade / dry-run mode (with `freqtrade trade`) - the UI will be available under the configured API port (by default `http://127.0.0.1:8080`).
+
+??? Note "Looking to contribute to freqUI?"
+    Developers should not use this method, but instead clone the corresponding use the method described in the [freqUI repository](https://github.com/freqtrade/frequi) to get the source-code of freqUI. A working installation of node will be required to build the frontend.
+
+!!! tip "freqUI is not required to run freqtrade"
+    freqUI is an optional component of freqtrade, and is not required to run the bot.
+    It is a frontend that can be used to monitor the bot and to interact with it - but freqtrade itself will work perfectly fine without it.
+
+## Configuration
+
+FreqUI does not have it's own configuration file - but assumes a working setup for the [rest-api](rest-api.md) is available.
+Please refer to the corresponding documentation page to get setup with freqUI
+
+## UI
+
+FreqUI is a modern, responsive web application that can be used to monitor and interact with your bot.
+
+FreqUI provides a light, as well as a dark theme.
+Themes can be easily switched via a prominent button at the top of the page.
+The theme of the screenshots on this page will adapt to the selected documentation Theme, so to see the dark (or light) version, please switch the theme of the Documentation.
+
+### Login
+
+The below screenshot shows the login screen of freqUI.
+
+![FreqUI - login](assets/frequi-login-CORS.png#only-dark)
+![FreqUI - login](assets/frequi-login-CORS-light.png#only-light)
+
+!!! Hint "CORS"
+    The Cors error shown in this screenshot is due to the fact that the UI is running on a different port than the API, and [CORS](#cors) has not been setup correctly yet.
+
+### Trade view
+
+The trade view allows you to visualize the trades that the bot is making and to interact with the bot.
+On this page, you can also interact with the bot by starting and stopping it and - if configured - force trade entries and exits.
+
+![FreqUI - trade view](assets/freqUI-trade-pane-dark.png#only-dark)
+![FreqUI - trade view](assets/freqUI-trade-pane-light.png#only-light)
+
+### Plot Configurator
+
+FreqUI Plots can be configured either via a `plot_config` configuration object in the strategy (which can be loaded via "from strategy" button) or via the UI.
+Multiple plot configurations can be created and switched at will - allowing for flexible, different views into your charts.
+
+The plot configuration can be accessed via the "Plot Configurator" (Cog icon) button in the top right corner of the trade view.
+
+![FreqUI - plot configuration](assets/freqUI-plot-configurator-dark.png#only-dark)
+![FreqUI - plot configuration](assets/freqUI-plot-configurator-light.png#only-light)
+
+### Settings
+
+
+Several UI related settings can be changed by accessing the settings page.
+
+Things you can change (among others):
+
+* Timezone of the UI
+* Visualization of open trades as part of the favicon (browser tab)
+* Candle colors (up/down -> red/green)
+* Enable / disable in-app notification types
+
+![FreqUI - Settings view](assets/frequi-settings-dark.png#only-dark)
+![FreqUI - Settings view](assets/frequi-settings-light.png#only-light)
+
+## Backtesting
+
+When freqtrade is started in [webserver mode](utils.md#webserver-mode) (freqtrade started with `freqtrade webserver`), the backtesting view becomes available.
+This view allows you to backtest strategies and visualize the results.
+
+You can also load and visualize previous backtest results, as well as compare the results with each other.
+
+![FreqUI - Backtesting](assets/freqUI-backtesting-dark.png#only-dark)
+![FreqUI - Backtesting](assets/freqUI-backtesting-light.png#only-light)
+
+
+--8<-- "includes/cors.md"

docs/freqai-feature-engineering.md

@@ -224,7 +224,7 @@ where $W_i$ is the weight of data point $i$ in a total set of $n$ data points. B
 
 ## Building the data pipeline
 
-By default, FreqAI builds a dynamic pipeline based on user congfiguration settings. The default settings are robust and designed to work with a variety of methods. These two steps are a `MinMaxScaler(-1,1)` and a `VarianceThreshold` which removes any column that has 0 variance. Users can activate other steps with more configuration parameters. For example if users add `use_SVM_to_remove_outliers: true` to the `freqai` config, then FreqAI will automatically add the [`SVMOutlierExtractor`](#identifying-outliers-using-a-support-vector-machine-svm) to the pipeline. Likewise, users can add `principal_component_analysis: true` to the `freqai` config to activate PCA. The [DissimilarityIndex](#identifying-outliers-with-the-dissimilarity-index-di) is activated with `DI_threshold: 1`. Finally, noise can also be added to the data with `noise_standard_deviation: 0.1`. Finally, users can add [DBSCAN](#identifying-outliers-with-dbscan) outlier removal with `use_DBSCAN_to_remove_outliers: true`.
+By default, FreqAI builds a dynamic pipeline based on user configuration settings. The default settings are robust and designed to work with a variety of methods. These two steps are a `MinMaxScaler(-1,1)` and a `VarianceThreshold` which removes any column that has 0 variance. Users can activate other steps with more configuration parameters. For example if users add `use_SVM_to_remove_outliers: true` to the `freqai` config, then FreqAI will automatically add the [`SVMOutlierExtractor`](#identifying-outliers-using-a-support-vector-machine-svm) to the pipeline. Likewise, users can add `principal_component_analysis: true` to the `freqai` config to activate PCA. The [DissimilarityIndex](#identifying-outliers-with-the-dissimilarity-index-di) is activated with `DI_threshold: 1`. Finally, noise can also be added to the data with `noise_standard_deviation: 0.1`. Finally, users can add [DBSCAN](#identifying-outliers-with-dbscan) outlier removal with `use_DBSCAN_to_remove_outliers: true`.
 
 !!! note "More information available"
     Please review the [parameter table](freqai-parameter-table.md) for more information on these parameters.
@@ -391,3 +391,18 @@ Given a number of data points $N$, and a distance $\varepsilon$, DBSCAN clusters
 ![dbscan](assets/freqai_dbscan.jpg)
 
 FreqAI uses `sklearn.cluster.DBSCAN` (details are available on scikit-learn's webpage [here](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html) (external website)) with `min_samples` ($N$) taken as 1/4 of the no. of time points (candles) in the feature set. `eps` ($\varepsilon$) is computed automatically as the elbow point in the *k-distance graph* computed from the nearest neighbors in the pairwise distances of all data points in the feature set.
+
+
+### Data dimensionality reduction with Principal Component Analysis
+
+You can reduce the dimensionality of your features by activating the principal_component_analysis in the config:
+
+```json
+    "freqai": {
+        "feature_parameters" : {
+            "principal_component_analysis": true
+        }
+    }
+```
+
+This will perform PCA on the features and reduce their dimensionality so that the explained variance of the data set is >= 0.999. Reducing data dimensionality makes training the model faster and hence allows for more up-to-date models.

docs/freqai-parameter-table.md

@@ -36,7 +36,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `weight_factor` | Weight training data points according to their recency (see details [here](freqai-feature-engineering.md#weighting-features-for-temporal-importance)). <br> **Datatype:** Positive float (typically < 1).
 | `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `feature_engineering_*()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN. <br> **Datatype:** Positive integer.
 | `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset. <br> **Datatype:** List of positive integers.
-| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`.
+| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`.
 | `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. Plot is stored in `user_data/models/<identifier>/sub-train-<COIN>_<timestamp>.html`. <br> **Datatype:** Integer. <br> Default: `0`.
 | `DI_threshold` | Activates the use of the Dissimilarity Index for outlier detection when set to > 0. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Positive float (typically < 1).
 | `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training dataset, as well as from incoming data points. See details about how it works [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm). <br> **Datatype:** Boolean.

docs/freqai-running.md

@@ -73,23 +73,26 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
 
 ---
 
-### Saving prediction data
+### Saving backtesting prediction data
 
 To allow for tweaking your strategy (**not** the features!), FreqAI will automatically save the predictions during backtesting so that they can be reused for future backtests and live runs using the same `identifier` model. This provides a performance enhancement geared towards enabling **high-level hyperopting** of entry/exit criteria.
 
-An additional directory called `backtesting_predictions`, which contains all the predictions stored in `hdf` format, will be created in the `unique-id` folder.
+An additional directory called `backtesting_predictions`, which contains all the predictions stored in `feather` format, will be created in the `unique-id` folder.
 
 To change your **features**, you **must** set a new `identifier` in the config to signal to FreqAI to train new models.
 
 To save the models generated during a particular backtest so that you can start a live deployment from one of them instead of training a new model, you must set `save_backtest_models` to `True` in the config.
 
+!!! Note
+    To ensure that the model can be reused, freqAI will call your strategy with a dataframe of length 1. 
+    If your strategy requires more data than this to generate the same features, you can't reuse backtest predictions for live deployment and need to update your `identifier` for each new backtest.
+
 ### Backtest live collected predictions
 
 FreqAI allow you to reuse live historic predictions through the backtest parameter `--freqai-backtest-live-models`. This can be useful when you want to reuse predictions generated in dry/run for comparison or other study.
 
 The `--timerange` parameter must not be informed, as it will be automatically calculated through the data in the historic predictions file.
 
-
 ### Downloading data to cover the full backtest period
 
 For live/dry deployments, FreqAI will download the necessary data automatically. However, to use backtesting functionality, you need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). You need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that there is a sufficient amount of training data *before* the start of the backtesting time range. The amount of additional data can be roughly estimated by moving the start date of the time range backwards by `train_period_days` and the `startup_candle_count` (see the [parameter table](freqai-parameter-table.md) for detailed descriptions of these parameters) from the beginning of the desired backtesting time range. 

docs/includes/cors.md

@@ -0,0 +1,43 @@
+## CORS
+
+This whole section is only necessary in cross-origin cases (where you multiple bot API's running on `localhost:8081`, `localhost:8082`, ...), and want to combine them into one FreqUI instance.
+
+??? info "Technical explanation"
+    All web-based front-ends are subject to [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) - Cross-Origin Resource Sharing.
+    Since most of the requests to the Freqtrade API must be authenticated, a proper CORS policy is key to avoid security problems.
+    Also, the standard disallows `*` CORS policies for requests with credentials, so this setting must be set appropriately.
+
+Users can allow access from different origin URL's to the bot API via the `CORS_origins` configuration setting.
+It consists of a list of allowed URL's that are allowed to consume resources from the bot's API.
+
+Assuming your application is deployed as `https://frequi.freqtrade.io/home/` - this would mean that the following configuration becomes necessary:
+
+```jsonc
+{
+    //...
+    "jwt_secret_key": "somethingrandom",
+    "CORS_origins": ["https://frequi.freqtrade.io"],
+    //...
+}
+```
+
+In the following (pretty common) case, FreqUI is accessible on `http://localhost:8080/trade` (this is what you see in your navbar when navigating to freqUI).
+![freqUI url](assets/frequi_url.png)
+
+The correct configuration for this case is `http://localhost:8080` - the main part of the URL including the port.
+
+```jsonc
+{
+    //...
+    "jwt_secret_key": "somethingrandom",
+    "CORS_origins": ["http://localhost:8080"],
+    //...
+}
+```
+
+!!! Tip "trailing Slash"
+    The trailing slash is not allowed in the `CORS_origins` configuration (e.g. `"http://localhots:8080/"`).
+    Such a configuration will not take effect, and the cors errors will remain.
+
+!!! Note
+    We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot.

docs/includes/pairlists.md

@@ -373,7 +373,7 @@ Filters low-value coins which would not allow setting stoplosses.
 
 Namely, pairs are blacklisted if a variance of one percent or more in the stop price would be caused by precision rounding on the exchange, i.e. `rounded(stop_price) <= rounded(stop_price * 0.99)`. The idea is to avoid coins with a value VERY close to their lower trading boundary, not allowing setting of proper stoploss.
 
-!!! Tip "PerformanceFilter is pointless for futures trading"
+!!! Tip "PrecisionFilter is pointless for futures trading"
     The above does not apply to shorts. And for longs, in theory the trade will be liquidated first.
 
 !!! Warning "Backtesting"

docs/index.md

@@ -41,6 +41,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 
 - [X] [Binance](https://www.binance.com/)
 - [X] [Bitmart](https://bitmart.com/)
+- [X] [BingX](https://bingx.com/invite/0EM9RX)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)

docs/installation.md

@@ -286,7 +286,7 @@ cd freqtrade
 #### Freqtrade install: Conda Environment
 
 ```bash
-conda create --name freqtrade python=3.11
+conda create --name freqtrade python=3.12
 ```
 
 !!! Note "Creating Conda Environment"

docs/plotting.md

@@ -2,6 +2,14 @@
 
 This page explains how to plot prices, indicators and profits.
 
+!!! Warning "Deprecated"
+    The commands described in this page (`plot-dataframe`, `plot-profit`) should be considered deprecated and are in maintenance mode.
+    This is mostly for the performance problems even medium sized plots can cause, but also because "store a file and open it in a browser" isn't very intuitive from a UI perspective.
+
+    While there are no immediate plans to remove them, they are not actively maintained - and may be removed short-term should major changes be required to keep them working.
+    
+    Please use [FreqUI](freq-ui.md) for plotting needs, which doesn't struggle with the same performance problems.
+
 ## Installation / Setup
 
 Plotting modules use the Plotly library. You can install / upgrade this by running the following command:

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.6
 mkdocs==1.6.0
-mkdocs-material==9.5.19
+mkdocs-material==9.5.29
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.8.1
-jinja2==3.1.3
+jinja2==3.1.4

docs/rest-api.md

@@ -1,16 +1,8 @@
-# REST API & FreqUI
+# REST API
 
 ## FreqUI
 
-Freqtrade provides a builtin webserver, which can serve [FreqUI](https://github.com/freqtrade/frequi), the freqtrade UI.
-
-By default, the UI is not included in the installation (except for docker images), and must be installed explicitly with `freqtrade install-ui`.
-This same command can also be used to update freqUI, should there be a new release.
-
-Once the bot is started in trade / dry-run mode (with `freqtrade trade`) - the UI will be available under the configured port below (usually `http://127.0.0.1:8080`).
-
-!!! Note "developers"
-    Developers should not use this method, but instead use the method described in the [freqUI repository](https://github.com/freqtrade/frequi) to get the source-code of freqUI.
+FreqUI now has it's own dedicated [documentation section](freq-ui.md) - please refer to that section for all information regarding the FreqUI.
 
 ## Configuration
 
@@ -126,6 +118,14 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
 freqtrade-client --config rest_config.json <command> [optional parameters]
 ```
 
+Commands with many arguments may require keyword arguments (for clarity) - which can be provided as follows:
+
+``` bash
+freqtrade-client --config rest_config.json forceenter BTC/USDT long enter_tag=GutFeeling
+```
+
+This method will work for all arguments - check the "show" command for a list of available parameters.
+
 ??? Note "Programmatic use"
     The `freqtrade-client` package (installable independent of freqtrade) can be used in your own scripts to interact with the freqtrade API.
     to do so, please use the following:
@@ -169,7 +169,7 @@ freqtrade-client --config rest_config.json <command> [optional parameters]
 | `delete_lock <lock_id>` | Deletes (disables) the lock by id.
 | `locks add <pair>, <until>, [side], [reason]` | Locks a pair until "until". (Until will be rounded up to the nearest timeframe).
 | `profit` | Display a summary of your profit/loss from close trades and some stats about your performance.
-| `forceexit <trade_id>` | Instantly exits the given trade  (Ignoring `minimum_roi`).
+| `forceexit <trade_id> [order_type] [amount]` | Instantly exits the given trade (ignoring `minimum_roi`), using the given order type ("market" or "limit", uses your config setting if not specified), and the chosen amount (full sell if not specified).
 | `forceexit all` | Instantly exits all open trades (Ignoring `minimum_roi`).
 | `forceenter <pair> [rate]` | Instantly enters the given pair. Rate is optional. (`force_entry_enable` must be set to True)
 | `forceenter <pair> <side> [rate]` | Instantly longs or shorts the given pair. Rate is optional. (`force_entry_enable` must be set to True)
@@ -488,42 +488,4 @@ Since the access token has a short timeout (15 min) - the `token/refresh` reques
 {"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"}
 ```
 
-### CORS
-
-This whole section is only necessary in cross-origin cases (where you multiple bot API's running on `localhost:8081`, `localhost:8082`, ...), and want to combine them into one FreqUI instance.
-
-??? info "Technical explanation"
-    All web-based front-ends are subject to [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) - Cross-Origin Resource Sharing.
-    Since most of the requests to the Freqtrade API must be authenticated, a proper CORS policy is key to avoid security problems.
-    Also, the standard disallows `*` CORS policies for requests with credentials, so this setting must be set appropriately.
-
-Users can allow access from different origin URL's to the bot API via the `CORS_origins` configuration setting.
-It consists of a list of allowed URL's that are allowed to consume resources from the bot's API.
-
-Assuming your application is deployed as `https://frequi.freqtrade.io/home/` - this would mean that the following configuration becomes necessary:
-
-```jsonc
-{
-    //...
-    "jwt_secret_key": "somethingrandom",
-    "CORS_origins": ["https://frequi.freqtrade.io"],
-    //...
-}
-```
-
-In the following (pretty common) case, FreqUI is accessible on `http://localhost:8080/trade` (this is what you see in your navbar when navigating to freqUI).
-![freqUI url](assets/frequi_url.png)
-
-The correct configuration for this case is `http://localhost:8080` - the main part of the URL including the port.
-
-```jsonc
-{
-    //...
-    "jwt_secret_key": "somethingrandom",
-    "CORS_origins": ["http://localhost:8080"],
-    //...
-}
-```
-
-!!! Note
-    We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot.
+--8<-- "includes/cors.md"

docs/sql_cheatsheet.md

@@ -1,6 +1,13 @@
 # SQL Helper
 
-This page contains some help if you want to edit your sqlite db.
+This page contains some help if you want to query your sqlite db.
+
+!!! Tip "Other Database systems"
+    To use other Database Systems like PostgreSQL or MariaDB, you can use the same queries, but you need to use the respective client for the database system. [Click here](advanced-setup.md#use-a-different-database-system) to learn how to setup a different database system with freqtrade.
+
+!!! Warning
+    If you are not familiar with SQL, you should be very careful when running queries on your database.  
+    Always make sure to have a backup of your database before running any queries.
 
 ## Install sqlite3
 
@@ -43,13 +50,25 @@ sqlite3
 .schema <table_name>
 ```
 
-## Get all trades in the table
+### Get all trades in the table
 
 ```sql
 SELECT * FROM trades;
 ```
 
-## Fix trade still open after a manual exit on the exchange
+## Destructive queries
+
+Queries that write to the database.
+These queries should usually not be necessary as freqtrade tries to handle all database operations itself - or exposes them via API or telegram commands.
+
+!!! Warning
+    Please make sure you have a backup of your database before running any of the below queries.
+
+!!! Danger
+    You should also **never** run any writing query (`update`, `insert`, `delete`) while a bot is connected to the database.
+    This can and will lead to data corruption - most likely, without the possibility of recovery.
+
+### Fix trade still open after a manual exit on the exchange
 
 !!! Warning
     Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, /forceexit <tradeid> should be used to accomplish the same thing.  
@@ -69,7 +88,7 @@ SET is_open=0,
 WHERE id=<trade_ID_to_update>;
 ```
 
-### Example
+#### Example
 
 ```sql
 UPDATE trades
@@ -82,7 +101,7 @@ SET is_open=0,
 WHERE id=31;
 ```
 
-## Remove trade from the database
+### Remove trade from the database
 
 !!! Tip "Use RPC Methods to delete trades"
     Consider using `/delete <tradeid>` via telegram or rest API. That's the recommended way to deleting trades.
@@ -100,39 +119,3 @@ DELETE FROM trades WHERE id = 31;
 
 !!! Warning
     This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
-
-## Use a different database system
-
-Freqtrade is using SQLAlchemy, which supports multiple different database systems. As such, a multitude of database systems should be supported.
-Freqtrade does not depend or install any additional database driver. Please refer to the [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) on installation instructions for the respective database systems.
-
-The following systems have been tested and are known to work with freqtrade:
-
-* sqlite (default)
-* PostgreSQL
-* MariaDB
-
-!!! Warning
-    By using one of the below database systems, you acknowledge that you know how to manage such a system. The freqtrade team will not provide any support with setup or maintenance (or backups) of the below database systems.
-
-### PostgreSQL
-
-Installation:
-`pip install psycopg2-binary`
-
-Usage:
-`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
-
-Freqtrade will automatically create the tables necessary upon startup.
-
-If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
-
-### MariaDB / MySQL
-
-Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
-
-Installation:
-`pip install pymysql`
-
-Usage:
-`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

docs/stoploss.md

@@ -30,6 +30,7 @@ The Order-type will be ignored if only one mode is available.
 |----------|-------------|
 | Binance  | limit |
 | Binance Futures  | market, limit |
+| Bingx  | market, limit |
 | HTX (former Huobi)    | limit |
 | kraken   | market, limit |
 | Gate     | limit |

docs/strategy-callbacks.md

@@ -165,7 +165,9 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
-Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+Returning `None` will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+`NaN` and `inf` values are considered invalid and will be ignored (identical to `None`).
+
 
 Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchangefreqtrade)).
 
@@ -467,7 +469,7 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
 
 ??? Example "Returning a stoploss using absolute price from the custom stoploss function"
 
-    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage)`.
+    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate=current_rate, is_short=trade.is_short, leverage=trade.leverage)`.
     For futures, we need to adjust the direction (up or down), as well as adjust for leverage, since the [`custom_stoploss`](strategy-callbacks.md#custom-stoploss) callback  returns the ["risk for this trade"](stoploss.md#stoploss-and-leverage) - not the relative price movement.
 
     ``` python
@@ -492,7 +494,8 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
             candle = dataframe.iloc[-1].squeeze()
             side = 1 if trade.is_short else -1
             return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), 
-                                          current_rate, is_short=trade.is_short,
+                                          current_rate=current_rate, 
+                                          is_short=trade.is_short,
                                           leverage=trade.leverage)
 
     ```

docs/trade-object.md

@@ -13,28 +13,28 @@ The following attributes / properties are available for each individual trade -
 
 |  Attribute | DataType | Description |
 |------------|-------------|-------------|
-`pair`| string | Pair of this trade
-`is_open`| boolean | Is the trade currently open, or has it been concluded
-`open_rate`| float | Rate this trade was entered at (Avg. entry rate in case of trade-adjustments)
-`close_rate`| float | Close rate - only set when is_open = False
-`stake_amount`| float | Amount in Stake (or Quote) currency.
-`amount`| float | Amount in Asset / Base currency that is currently owned.
-`open_date`| datetime | Timestamp when trade was opened **use `open_date_utc` instead**
-`open_date_utc`| datetime | Timestamp when trade was opened - in UTC
-`close_date`| datetime | Timestamp when trade was closed **use `close_date_utc` instead**
-`close_date_utc`| datetime | Timestamp when trade was closed - in UTC
-`close_profit`| float | Relative profit at the time of trade closure. `0.01` == 1%
-`close_profit_abs`| float | Absolute profit (in stake currency) at the time of trade closure.
-`leverage` | float | Leverage used for this trade - defaults to 1.0 in spot markets.
-`enter_tag`| string | Tag provided on entry via the `enter_tag` column in the dataframe
-`is_short` | boolean | True for short trades, False otherwise
-`orders` | Order[] | List of order objects attached to this trade (includes both filled and cancelled orders)
-`date_last_filled_utc` | datetime | Time of the last filled order
-`entry_side` | "buy" / "sell" | Order Side the trade was entered
-`exit_side` | "buy" / "sell" | Order Side that will result in a trade exit / position reduction.
-`trade_direction` | "long" / "short" | Trade direction in text - long or short.
-`nr_of_successful_entries` | int | Number of successful (filled) entry orders
-`nr_of_successful_exits` | int | Number of successful (filled) exit orders
+| `pair` | string | Pair of this trade. |
+| `is_open` | boolean | Is the trade currently open, or has it been concluded. |
+| `open_rate` | float | Rate this trade was entered at (Avg. entry rate in case of trade-adjustments). |
+| `close_rate` | float | Close rate - only set when is_open = False. |
+| `stake_amount` | float | Amount in Stake (or Quote) currency. |
+| `amount` | float | Amount in Asset / Base currency that is currently owned. |
+| `open_date` | datetime | Timestamp when trade was opened **use `open_date_utc` instead** |
+| `open_date_utc` | datetime | Timestamp when trade was opened - in UTC. |
+| `close_date` | datetime | Timestamp when trade was closed **use `close_date_utc` instead** |
+| `close_date_utc` | datetime | Timestamp when trade was closed - in UTC. |
+| `close_profit` | float | Relative profit at the time of trade closure. `0.01` == 1% |
+| `close_profit_abs` | float | Absolute profit (in stake currency) at the time of trade closure. |
+| `leverage` | float | Leverage used for this trade - defaults to 1.0 in spot markets. |
+| `enter_tag` | string | Tag provided on entry via the `enter_tag` column in the dataframe. |
+| `is_short` | boolean | True for short trades, False otherwise. |
+| `orders` | Order[] | List of order objects attached to this trade (includes both filled and cancelled orders). |
+| `date_last_filled_utc` | datetime | Time of the last filled order. |
+| `entry_side` | "buy" / "sell" | Order Side the trade was entered. |
+| `exit_side` | "buy" / "sell" | Order Side that will result in a trade exit / position reduction. |
+| `trade_direction` | "long" / "short" | Trade direction in text - long or short. |
+| `nr_of_successful_entries` | int | Number of successful (filled) entry orders. |
+| `nr_of_successful_exits` | int | Number of successful (filled) exit orders. |
 
 ## Class methods
 

docs/utils.md

@@ -488,7 +488,7 @@ freqtrade test-pairlist --config config.json --quote USDT BTC
 
 `freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks.
 
-Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems.
+Please refer to the [corresponding documentation](advanced-setup.md#use-a-different-database-system) to learn about requirements for different database systems.
 
 ```
 usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH]

docs/windows_installation.md

@@ -5,6 +5,30 @@ We **strongly** recommend that Windows users use [Docker](docker_quickstart.md)
 If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work.
 Otherwise, please follow the instructions below.
 
+All instructions assume that python 3.9+ is installed and available.
+
+## Clone the git repository
+
+First of all clone the repository by running:
+
+``` powershell
+git clone https://github.com/freqtrade/freqtrade.git
+```
+
+Now, choose your installation method, either automatically via script (recommended) or manually following the corresponding instructions.
+
+## Install freqtrade automatically
+
+### Run the installation script
+
+The script will ask you a few questions to determine which parts should be installed.
+
+```powershell
+Set-ExecutionPolicy -ExecutionPolicy Bypass
+cd freqtrade
+. .\setup.ps1
+```
+
 ## Install freqtrade manually
 
 !!! Note "64bit Python version"
@@ -14,17 +38,11 @@ Otherwise, please follow the instructions below.
 !!! Hint
     Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#installation-with-conda) in the documentation for more information.
 
-### 1. Clone the git repository
-
-```bash
-git clone https://github.com/freqtrade/freqtrade.git
-```
-
-### 2. Install ta-lib
+### Install ta-lib
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10, 3.11 and 3.12) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.
 
 Other versions must be downloaded from the above link.

freqtrade/__init__.py

@@ -1,22 +1,34 @@
-""" Freqtrade bot """
-__version__ = '2024.4'
+"""Freqtrade bot"""
 
-if 'dev' in __version__:
+__version__ = "2024.7"
+
+if "dev" in __version__:
     from pathlib import Path
+
     try:
-        import subprocess
+        import subprocess  # noqa: S404
+
         freqtrade_basedir = Path(__file__).parent
 
-        __version__ = __version__ + '-' + subprocess.check_output(
-            ['git', 'log', '--format="%h"', '-n 1'],
-            stderr=subprocess.DEVNULL, cwd=freqtrade_basedir).decode("utf-8").rstrip().strip('"')
+        __version__ = (
+            __version__
+            + "-"
+            + subprocess.check_output(
+                ["git", "log", '--format="%h"', "-n 1"],
+                stderr=subprocess.DEVNULL,
+                cwd=freqtrade_basedir,
+            )
+            .decode("utf-8")
+            .rstrip()
+            .strip('"')
+        )
 
     except Exception:  # pragma: no cover
         # git not available, ignore
         try:
             # Try Fallback to freqtrade_commit file (created by CI while building docker image)
-            versionfile = Path('./freqtrade_commit')
+            versionfile = Path("./freqtrade_commit")
             if versionfile.is_file():
                 __version__ = f"docker-{__version__}-{versionfile.read_text()[:8]}"
-        except Exception:
+        except Exception:  # noqa: S110
             pass

freqtrade/__main__.py

@@ -9,5 +9,5 @@ To launch Freqtrade as a module
 from freqtrade import main
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main.main()

freqtrade/commands/__init__.py

@@ -6,22 +6,39 @@ Contains all start-commands, subcommands and CLI Interface creation.
 Note: Be careful with file-scoped imports in these subfiles.
     as they are parsed on startup, nothing containing optional modules should be loaded.
 """
+
 from freqtrade.commands.analyze_commands import start_analysis_entries_exits
 from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config, start_show_config
-from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
-                                              start_download_data, start_list_data)
+from freqtrade.commands.data_commands import (
+    start_convert_data,
+    start_convert_trades,
+    start_download_data,
+    start_list_data,
+)
 from freqtrade.commands.db_commands import start_convert_db
-from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
-                                                start_new_strategy)
+from freqtrade.commands.deploy_commands import (
+    start_create_userdir,
+    start_install_ui,
+    start_new_strategy,
+)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
-from freqtrade.commands.list_commands import (start_list_exchanges, start_list_freqAI_models,
-                                              start_list_markets, start_list_strategies,
-                                              start_list_timeframes, start_show_trades)
-from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show,
-                                                  start_edge, start_hyperopt,
-                                                  start_lookahead_analysis,
-                                                  start_recursive_analysis)
+from freqtrade.commands.list_commands import (
+    start_list_exchanges,
+    start_list_freqAI_models,
+    start_list_markets,
+    start_list_strategies,
+    start_list_timeframes,
+    start_show_trades,
+)
+from freqtrade.commands.optimize_commands import (
+    start_backtesting,
+    start_backtesting_show,
+    start_edge,
+    start_hyperopt,
+    start_lookahead_analysis,
+    start_recursive_analysis,
+)
 from freqtrade.commands.pairlist_commands import start_test_pairlist
 from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
 from freqtrade.commands.strategy_utils_commands import start_strategy_update

freqtrade/commands/analyze_commands.py

@@ -20,25 +20,25 @@ def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[s
     config = setup_utils_configuration(args, method)
 
     no_unlimited_runmodes = {
-        RunMode.BACKTEST: 'backtesting',
+        RunMode.BACKTEST: "backtesting",
     }
     if method in no_unlimited_runmodes.keys():
         from freqtrade.data.btanalysis import get_latest_backtest_filename
 
-        if 'exportfilename' in config:
-            if config['exportfilename'].is_dir():
-                btfile = Path(get_latest_backtest_filename(config['exportfilename']))
+        if "exportfilename" in config:
+            if config["exportfilename"].is_dir():
+                btfile = Path(get_latest_backtest_filename(config["exportfilename"]))
                 signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl"
             else:
-                if config['exportfilename'].exists():
-                    btfile = Path(config['exportfilename'])
+                if config["exportfilename"].exists():
+                    btfile = Path(config["exportfilename"])
                     signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl"
                 else:
                     raise ConfigurationError(f"{config['exportfilename']} does not exist.")
         else:
-            raise ConfigurationError('exportfilename not in config.')
+            raise ConfigurationError("exportfilename not in config.")
 
-        if (not Path(signals_file).exists()):
+        if not Path(signals_file).exists():
             raise OperationalException(
                 f"Cannot find latest backtest signals file: {signals_file}."
                 "Run backtesting with `--export signals`."
@@ -58,6 +58,6 @@ def start_analysis_entries_exits(args: Dict[str, Any]) -> None:
     # Initialize configuration
     config = setup_analyze_configuration(args, RunMode.BACKTEST)
 
-    logger.info('Starting freqtrade in analysis mode')
+    logger.info("Starting freqtrade in analysis mode")
 
     process_entry_exit_reasons(config)

freqtrade/commands/arguments.py

@@ -1,10 +1,11 @@
 """
 This module contains the argument manager class
 """
-import argparse
+
+from argparse import ArgumentParser, Namespace, _ArgumentGroup
 from functools import partial
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 from freqtrade.constants import DEFAULT_CONFIG
@@ -12,35 +13,72 @@ from freqtrade.constants import DEFAULT_CONFIG
 
 ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
 
-ARGS_STRATEGY = ["strategy", "strategy_path", "recursive_strategy_search", "freqaimodel",
-                 "freqaimodel_path"]
+ARGS_STRATEGY = [
+    "strategy",
+    "strategy_path",
+    "recursive_strategy_search",
+    "freqaimodel",
+    "freqaimodel_path",
+]
 
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
 
 ARGS_WEBSERVER: List[str] = []
 
-ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
-                        "max_open_trades", "stake_amount", "fee", "pairs"]
+ARGS_COMMON_OPTIMIZE = [
+    "timeframe",
+    "timerange",
+    "dataformat_ohlcv",
+    "max_open_trades",
+    "stake_amount",
+    "fee",
+    "pairs",
+]
 
-ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
-                                        "enable_protections", "dry_run_wallet", "timeframe_detail",
-                                        "strategy_list", "export", "exportfilename",
-                                        "backtest_breakdown", "backtest_cache",
-                                        "freqai_backtest_live_models"]
+ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + [
+    "position_stacking",
+    "use_max_market_positions",
+    "enable_protections",
+    "dry_run_wallet",
+    "timeframe_detail",
+    "strategy_list",
+    "export",
+    "exportfilename",
+    "backtest_breakdown",
+    "backtest_cache",
+    "freqai_backtest_live_models",
+]
 
-ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
-                                        "position_stacking", "use_max_market_positions",
-                                        "enable_protections", "dry_run_wallet", "timeframe_detail",
-                                        "epochs", "spaces", "print_all",
-                                        "print_colorized", "print_json", "hyperopt_jobs",
-                                        "hyperopt_random_state", "hyperopt_min_trades",
-                                        "hyperopt_loss", "disableparamexport",
-                                        "hyperopt_ignore_missing_space", "analyze_per_epoch"]
+ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + [
+    "hyperopt",
+    "hyperopt_path",
+    "position_stacking",
+    "use_max_market_positions",
+    "enable_protections",
+    "dry_run_wallet",
+    "timeframe_detail",
+    "epochs",
+    "spaces",
+    "print_all",
+    "print_colorized",
+    "print_json",
+    "hyperopt_jobs",
+    "hyperopt_random_state",
+    "hyperopt_min_trades",
+    "hyperopt_loss",
+    "disableparamexport",
+    "hyperopt_ignore_missing_space",
+    "analyze_per_epoch",
+]
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
-ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized",
-                        "recursive_strategy_search"]
+ARGS_LIST_STRATEGIES = [
+    "strategy_path",
+    "print_one_column",
+    "print_colorized",
+    "recursive_strategy_search",
+]
 
 ARGS_LIST_FREQAIMODELS = ["freqaimodel_path", "print_one_column", "print_colorized"]
 
@@ -52,12 +90,27 @@ ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"]
 
 ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
 
-ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column",
-                   "print_csv", "base_currencies", "quote_currencies", "list_pairs_all",
-                   "trading_mode"]
+ARGS_LIST_PAIRS = [
+    "exchange",
+    "print_list",
+    "list_pairs_print_json",
+    "print_one_column",
+    "print_csv",
+    "base_currencies",
+    "quote_currencies",
+    "list_pairs_all",
+    "trading_mode",
+]
 
-ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies",
-                      "print_one_column", "list_pairs_print_json", "exchange"]
+ARGS_TEST_PAIRLIST = [
+    "user_data_dir",
+    "verbosity",
+    "config",
+    "quote_currencies",
+    "print_one_column",
+    "list_pairs_print_json",
+    "exchange",
+]
 
 ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
 
@@ -70,22 +123,59 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase",
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
 
-ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades",
-                       "trading_mode"]
+ARGS_CONVERT_TRADES = [
+    "pairs",
+    "timeframes",
+    "exchange",
+    "dataformat_ohlcv",
+    "dataformat_trades",
+    "trading_mode",
+]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
-                      "timerange", "download_trades", "exchange", "timeframes",
-                      "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode",
-                      "prepend_data"]
+ARGS_DOWNLOAD_DATA = [
+    "pairs",
+    "pairs_file",
+    "days",
+    "new_pairs_days",
+    "include_inactive",
+    "timerange",
+    "download_trades",
+    "convert_trades",
+    "exchange",
+    "timeframes",
+    "erase",
+    "dataformat_ohlcv",
+    "dataformat_trades",
+    "trading_mode",
+    "prepend_data",
+]
 
-ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
-                       "db_url", "trade_source", "export", "exportfilename",
-                       "timerange", "timeframe", "no_trades"]
+ARGS_PLOT_DATAFRAME = [
+    "pairs",
+    "indicators1",
+    "indicators2",
+    "plot_limit",
+    "db_url",
+    "trade_source",
+    "export",
+    "exportfilename",
+    "timerange",
+    "timeframe",
+    "no_trades",
+]
 
-ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
-                    "trade_source", "timeframe", "plot_auto_open", ]
+ARGS_PLOT_PROFIT = [
+    "pairs",
+    "timerange",
+    "export",
+    "exportfilename",
+    "db_url",
+    "trade_source",
+    "timeframe",
+    "plot_auto_open",
+]
 
 ARGS_CONVERT_DB = ["db_url", "db_url_from"]
 
@@ -93,39 +183,84 @@ ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"]
 
 ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
 
-ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
-                      "hyperopt_list_min_trades", "hyperopt_list_max_trades",
-                      "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time",
-                      "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit",
-                      "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit",
-                      "hyperopt_list_min_objective", "hyperopt_list_max_objective",
-                      "print_colorized", "print_json", "hyperopt_list_no_details",
-                      "hyperoptexportfilename", "export_csv"]
+ARGS_HYPEROPT_LIST = [
+    "hyperopt_list_best",
+    "hyperopt_list_profitable",
+    "hyperopt_list_min_trades",
+    "hyperopt_list_max_trades",
+    "hyperopt_list_min_avg_time",
+    "hyperopt_list_max_avg_time",
+    "hyperopt_list_min_avg_profit",
+    "hyperopt_list_max_avg_profit",
+    "hyperopt_list_min_total_profit",
+    "hyperopt_list_max_total_profit",
+    "hyperopt_list_min_objective",
+    "hyperopt_list_max_objective",
+    "print_colorized",
+    "print_json",
+    "hyperopt_list_no_details",
+    "hyperoptexportfilename",
+    "export_csv",
+]
 
-ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
-                      "print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
-                      "disableparamexport", "backtest_breakdown"]
+ARGS_HYPEROPT_SHOW = [
+    "hyperopt_list_best",
+    "hyperopt_list_profitable",
+    "hyperopt_show_index",
+    "print_json",
+    "hyperoptexportfilename",
+    "hyperopt_show_no_header",
+    "disableparamexport",
+    "backtest_breakdown",
+]
 
-ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason_list",
-                              "exit_reason_list", "indicator_list", "timerange",
-                              "analysis_rejected", "analysis_to_csv", "analysis_csv_path"]
+ARGS_ANALYZE_ENTRIES_EXITS = [
+    "exportfilename",
+    "analysis_groups",
+    "enter_reason_list",
+    "exit_reason_list",
+    "indicator_list",
+    "timerange",
+    "analysis_rejected",
+    "analysis_to_csv",
+    "analysis_csv_path",
+]
 
-NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
-                    "list-markets", "list-pairs", "list-strategies", "list-freqaimodels",
-                    "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter",
-                    "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv",
-                    "strategy-updater"]
-
-NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
 
 ARGS_STRATEGY_UPDATER = ["strategy_list", "strategy_path", "recursive_strategy_search"]
 
 ARGS_LOOKAHEAD_ANALYSIS = [
-    a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", 'cache')
-    ] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
+    a
+    for a in ARGS_BACKTEST
+    if a
+    not in ("position_stacking", "use_max_market_positions", "backtest_cache", "backtest_breakdown")
+] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
 
 ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
 
+# Command level configs - keep at the bottom of the above definitions
+NO_CONF_REQURIED = [
+    "convert-data",
+    "convert-trade-data",
+    "download-data",
+    "list-timeframes",
+    "list-markets",
+    "list-pairs",
+    "list-strategies",
+    "list-freqaimodels",
+    "list-data",
+    "hyperopt-list",
+    "hyperopt-show",
+    "backtest-filter",
+    "plot-dataframe",
+    "plot-profit",
+    "show-trades",
+    "trades-to-ohlcv",
+    "strategy-updater",
+]
+
+NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
+
 
 class Arguments:
     """
@@ -134,7 +269,7 @@ class Arguments:
 
     def __init__(self, args: Optional[List[str]]) -> None:
         self.args = args
-        self._parsed_arg: Optional[argparse.Namespace] = None
+        self._parsed_arg: Optional[Namespace] = None
 
     def get_parsed_arg(self) -> Dict[str, Any]:
         """
@@ -147,7 +282,7 @@ class Arguments:
 
         return vars(self._parsed_arg)
 
-    def _parse_args(self) -> argparse.Namespace:
+    def _parse_args(self) -> Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -156,14 +291,14 @@ class Arguments:
         # Workaround issue in argparse with action='append' and default value
         # (see https://bugs.python.org/issue16399)
         # Allow no-config for certain commands (like downloading / plotting)
-        if ('config' in parsed_arg and parsed_arg.config is None):
-            conf_required = ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED)
+        if "config" in parsed_arg and parsed_arg.config is None:
+            conf_required = "command" in parsed_arg and parsed_arg.command in NO_CONF_REQURIED
 
-            if 'user_data_dir' in parsed_arg and parsed_arg.user_data_dir is not None:
+            if "user_data_dir" in parsed_arg and parsed_arg.user_data_dir is not None:
                 user_dir = parsed_arg.user_data_dir
             else:
                 # Default case
-                user_dir = 'user_data'
+                user_dir = "user_data"
                 # Try loading from "user_data/config.json"
             cfgfile = Path(user_dir) / DEFAULT_CONFIG
             if cfgfile.is_file():
@@ -176,8 +311,9 @@ class Arguments:
 
         return parsed_arg
 
-    def _build_args(self, optionlist, parser):
-
+    def _build_args(
+        self, optionlist: List[str], parser: Union[ArgumentParser, _ArgumentGroup]
+    ) -> None:
         for val in optionlist:
             opt = AVAILABLE_CLI_OPTIONS[val]
             parser.add_argument(*opt.cli, dest=val, **opt.kwargs)
@@ -188,53 +324,71 @@ class Arguments:
         :return: None
         """
         # Build shared arguments (as group Common Options)
-        _common_parser = argparse.ArgumentParser(add_help=False)
+        _common_parser = ArgumentParser(add_help=False)
         group = _common_parser.add_argument_group("Common arguments")
         self._build_args(optionlist=ARGS_COMMON, parser=group)
 
-        _strategy_parser = argparse.ArgumentParser(add_help=False)
+        _strategy_parser = ArgumentParser(add_help=False)
         strategy_group = _strategy_parser.add_argument_group("Strategy arguments")
         self._build_args(optionlist=ARGS_STRATEGY, parser=strategy_group)
 
         # Build main command
-        self.parser = argparse.ArgumentParser(
-            prog="freqtrade",
-            description='Free, open source crypto trading bot'
+        self.parser = ArgumentParser(
+            prog="freqtrade", description="Free, open source crypto trading bot"
         )
-        self._build_args(optionlist=['version'], parser=self.parser)
+        self._build_args(optionlist=["version"], parser=self.parser)
 
-        from freqtrade.commands import (start_analysis_entries_exits, start_backtesting,
-                                        start_backtesting_show, start_convert_data,
-                                        start_convert_db, start_convert_trades,
-                                        start_create_userdir, start_download_data, start_edge,
-                                        start_hyperopt, start_hyperopt_list, start_hyperopt_show,
-                                        start_install_ui, start_list_data, start_list_exchanges,
-                                        start_list_freqAI_models, start_list_markets,
-                                        start_list_strategies, start_list_timeframes,
-                                        start_lookahead_analysis, start_new_config,
-                                        start_new_strategy, start_plot_dataframe, start_plot_profit,
-                                        start_recursive_analysis, start_show_config,
-                                        start_show_trades, start_strategy_update,
-                                        start_test_pairlist, start_trading, start_webserver)
+        from freqtrade.commands import (
+            start_analysis_entries_exits,
+            start_backtesting,
+            start_backtesting_show,
+            start_convert_data,
+            start_convert_db,
+            start_convert_trades,
+            start_create_userdir,
+            start_download_data,
+            start_edge,
+            start_hyperopt,
+            start_hyperopt_list,
+            start_hyperopt_show,
+            start_install_ui,
+            start_list_data,
+            start_list_exchanges,
+            start_list_freqAI_models,
+            start_list_markets,
+            start_list_strategies,
+            start_list_timeframes,
+            start_lookahead_analysis,
+            start_new_config,
+            start_new_strategy,
+            start_plot_dataframe,
+            start_plot_profit,
+            start_recursive_analysis,
+            start_show_config,
+            start_show_trades,
+            start_strategy_update,
+            start_test_pairlist,
+            start_trading,
+            start_webserver,
+        )
 
-        subparsers = self.parser.add_subparsers(dest='command',
-                                                # Use custom message when no subhandler is added
-                                                # shown from `main.py`
-                                                # required=True
-                                                )
+        subparsers = self.parser.add_subparsers(
+            dest="command",
+            # Use custom message when no subhandler is added
+            # shown from `main.py`
+            # required=True
+        )
 
         # Add trade subcommand
         trade_cmd = subparsers.add_parser(
-            'trade',
-            help='Trade module.',
-            parents=[_common_parser, _strategy_parser]
+            "trade", help="Trade module.", parents=[_common_parser, _strategy_parser]
         )
         trade_cmd.set_defaults(func=start_trading)
         self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd)
 
         # add create-userdir subcommand
         create_userdir_cmd = subparsers.add_parser(
-            'create-userdir',
+            "create-userdir",
             help="Create user-data directory.",
         )
         create_userdir_cmd.set_defaults(func=start_create_userdir)
@@ -242,7 +396,7 @@ class Arguments:
 
         # add new-config subcommand
         build_config_cmd = subparsers.add_parser(
-            'new-config',
+            "new-config",
             help="Create new config",
         )
         build_config_cmd.set_defaults(func=start_new_config)
@@ -250,7 +404,7 @@ class Arguments:
 
         # add show-config subcommand
         show_config_cmd = subparsers.add_parser(
-            'show-config',
+            "show-config",
             help="Show resolved config",
         )
         show_config_cmd.set_defaults(func=start_show_config)
@@ -258,7 +412,7 @@ class Arguments:
 
         # add new-strategy subcommand
         build_strategy_cmd = subparsers.add_parser(
-            'new-strategy',
+            "new-strategy",
             help="Create new strategy",
         )
         build_strategy_cmd.set_defaults(func=start_new_strategy)
@@ -266,8 +420,8 @@ class Arguments:
 
         # Add download-data subcommand
         download_data_cmd = subparsers.add_parser(
-            'download-data',
-            help='Download backtesting data.',
+            "download-data",
+            help="Download backtesting data.",
             parents=[_common_parser],
         )
         download_data_cmd.set_defaults(func=start_download_data)
@@ -275,8 +429,8 @@ class Arguments:
 
         # Add convert-data subcommand
         convert_data_cmd = subparsers.add_parser(
-            'convert-data',
-            help='Convert candle (OHLCV) data from one format to another.',
+            "convert-data",
+            help="Convert candle (OHLCV) data from one format to another.",
             parents=[_common_parser],
         )
         convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True))
@@ -284,8 +438,8 @@ class Arguments:
 
         # Add convert-trade-data subcommand
         convert_trade_data_cmd = subparsers.add_parser(
-            'convert-trade-data',
-            help='Convert trade data from one format to another.',
+            "convert-trade-data",
+            help="Convert trade data from one format to another.",
             parents=[_common_parser],
         )
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
@@ -293,8 +447,8 @@ class Arguments:
 
         # Add trades-to-ohlcv subcommand
         convert_trade_data_cmd = subparsers.add_parser(
-            'trades-to-ohlcv',
-            help='Convert trade data to OHLCV data.',
+            "trades-to-ohlcv",
+            help="Convert trade data to OHLCV data.",
             parents=[_common_parser],
         )
         convert_trade_data_cmd.set_defaults(func=start_convert_trades)
@@ -302,8 +456,8 @@ class Arguments:
 
         # Add list-data subcommand
         list_data_cmd = subparsers.add_parser(
-            'list-data',
-            help='List downloaded data.',
+            "list-data",
+            help="List downloaded data.",
             parents=[_common_parser],
         )
         list_data_cmd.set_defaults(func=start_list_data)
@@ -311,17 +465,15 @@ class Arguments:
 
         # Add backtesting subcommand
         backtesting_cmd = subparsers.add_parser(
-            'backtesting',
-            help='Backtesting module.',
-            parents=[_common_parser, _strategy_parser]
+            "backtesting", help="Backtesting module.", parents=[_common_parser, _strategy_parser]
         )
         backtesting_cmd.set_defaults(func=start_backtesting)
         self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
 
         # Add backtesting-show subcommand
         backtesting_show_cmd = subparsers.add_parser(
-            'backtesting-show',
-            help='Show past Backtest results',
+            "backtesting-show",
+            help="Show past Backtest results",
             parents=[_common_parser],
         )
         backtesting_show_cmd.set_defaults(func=start_backtesting_show)
@@ -329,26 +481,22 @@ class Arguments:
 
         # Add backtesting analysis subcommand
         analysis_cmd = subparsers.add_parser(
-            'backtesting-analysis',
-            help='Backtest Analysis module.',
-            parents=[_common_parser]
+            "backtesting-analysis", help="Backtest Analysis module.", parents=[_common_parser]
         )
         analysis_cmd.set_defaults(func=start_analysis_entries_exits)
         self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
 
         # Add edge subcommand
         edge_cmd = subparsers.add_parser(
-            'edge',
-            help='Edge module.',
-            parents=[_common_parser, _strategy_parser]
+            "edge", help="Edge module.", parents=[_common_parser, _strategy_parser]
         )
         edge_cmd.set_defaults(func=start_edge)
         self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd)
 
         # Add hyperopt subcommand
         hyperopt_cmd = subparsers.add_parser(
-            'hyperopt',
-            help='Hyperopt module.',
+            "hyperopt",
+            help="Hyperopt module.",
             parents=[_common_parser, _strategy_parser],
         )
         hyperopt_cmd.set_defaults(func=start_hyperopt)
@@ -356,8 +504,8 @@ class Arguments:
 
         # Add hyperopt-list subcommand
         hyperopt_list_cmd = subparsers.add_parser(
-            'hyperopt-list',
-            help='List Hyperopt results',
+            "hyperopt-list",
+            help="List Hyperopt results",
             parents=[_common_parser],
         )
         hyperopt_list_cmd.set_defaults(func=start_hyperopt_list)
@@ -365,8 +513,8 @@ class Arguments:
 
         # Add hyperopt-show subcommand
         hyperopt_show_cmd = subparsers.add_parser(
-            'hyperopt-show',
-            help='Show details of Hyperopt results',
+            "hyperopt-show",
+            help="Show details of Hyperopt results",
             parents=[_common_parser],
         )
         hyperopt_show_cmd.set_defaults(func=start_hyperopt_show)
@@ -374,8 +522,8 @@ class Arguments:
 
         # Add list-exchanges subcommand
         list_exchanges_cmd = subparsers.add_parser(
-            'list-exchanges',
-            help='Print available exchanges.',
+            "list-exchanges",
+            help="Print available exchanges.",
             parents=[_common_parser],
         )
         list_exchanges_cmd.set_defaults(func=start_list_exchanges)
@@ -383,8 +531,8 @@ class Arguments:
 
         # Add list-markets subcommand
         list_markets_cmd = subparsers.add_parser(
-            'list-markets',
-            help='Print markets on exchange.',
+            "list-markets",
+            help="Print markets on exchange.",
             parents=[_common_parser],
         )
         list_markets_cmd.set_defaults(func=partial(start_list_markets, pairs_only=False))
@@ -392,8 +540,8 @@ class Arguments:
 
         # Add list-pairs subcommand
         list_pairs_cmd = subparsers.add_parser(
-            'list-pairs',
-            help='Print pairs on exchange.',
+            "list-pairs",
+            help="Print pairs on exchange.",
             parents=[_common_parser],
         )
         list_pairs_cmd.set_defaults(func=partial(start_list_markets, pairs_only=True))
@@ -401,8 +549,8 @@ class Arguments:
 
         # Add list-strategies subcommand
         list_strategies_cmd = subparsers.add_parser(
-            'list-strategies',
-            help='Print available strategies.',
+            "list-strategies",
+            help="Print available strategies.",
             parents=[_common_parser],
         )
         list_strategies_cmd.set_defaults(func=start_list_strategies)
@@ -410,8 +558,8 @@ class Arguments:
 
         # Add list-freqAI Models subcommand
         list_freqaimodels_cmd = subparsers.add_parser(
-            'list-freqaimodels',
-            help='Print available freqAI models.',
+            "list-freqaimodels",
+            help="Print available freqAI models.",
             parents=[_common_parser],
         )
         list_freqaimodels_cmd.set_defaults(func=start_list_freqAI_models)
@@ -419,8 +567,8 @@ class Arguments:
 
         # Add list-timeframes subcommand
         list_timeframes_cmd = subparsers.add_parser(
-            'list-timeframes',
-            help='Print available timeframes for the exchange.',
+            "list-timeframes",
+            help="Print available timeframes for the exchange.",
             parents=[_common_parser],
         )
         list_timeframes_cmd.set_defaults(func=start_list_timeframes)
@@ -428,8 +576,8 @@ class Arguments:
 
         # Add show-trades subcommand
         show_trades = subparsers.add_parser(
-            'show-trades',
-            help='Show trades.',
+            "show-trades",
+            help="Show trades.",
             parents=[_common_parser],
         )
         show_trades.set_defaults(func=start_show_trades)
@@ -437,8 +585,8 @@ class Arguments:
 
         # Add test-pairlist subcommand
         test_pairlist_cmd = subparsers.add_parser(
-            'test-pairlist',
-            help='Test your pairlist configuration.',
+            "test-pairlist",
+            help="Test your pairlist configuration.",
         )
         test_pairlist_cmd.set_defaults(func=start_test_pairlist)
         self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd)
@@ -453,16 +601,16 @@ class Arguments:
 
         # Add install-ui subcommand
         install_ui_cmd = subparsers.add_parser(
-            'install-ui',
-            help='Install FreqUI',
+            "install-ui",
+            help="Install FreqUI",
         )
         install_ui_cmd.set_defaults(func=start_install_ui)
         self._build_args(optionlist=ARGS_INSTALL_UI, parser=install_ui_cmd)
 
         # Add Plotting subcommand
         plot_dataframe_cmd = subparsers.add_parser(
-            'plot-dataframe',
-            help='Plot candles with indicators.',
+            "plot-dataframe",
+            help="Plot candles with indicators.",
             parents=[_common_parser, _strategy_parser],
         )
         plot_dataframe_cmd.set_defaults(func=start_plot_dataframe)
@@ -470,8 +618,8 @@ class Arguments:
 
         # Plot profit
         plot_profit_cmd = subparsers.add_parser(
-            'plot-profit',
-            help='Generate plot showing profits.',
+            "plot-profit",
+            help="Generate plot showing profits.",
             parents=[_common_parser, _strategy_parser],
         )
         plot_profit_cmd.set_defaults(func=start_plot_profit)
@@ -479,40 +627,36 @@ class Arguments:
 
         # Add webserver subcommand
         webserver_cmd = subparsers.add_parser(
-            'webserver',
-            help='Webserver module.',
-            parents=[_common_parser]
+            "webserver", help="Webserver module.", parents=[_common_parser]
         )
         webserver_cmd.set_defaults(func=start_webserver)
         self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
 
         # Add strategy_updater subcommand
         strategy_updater_cmd = subparsers.add_parser(
-            'strategy-updater',
-            help='updates outdated strategy files to the current version',
-            parents=[_common_parser]
+            "strategy-updater",
+            help="updates outdated strategy files to the current version",
+            parents=[_common_parser],
         )
         strategy_updater_cmd.set_defaults(func=start_strategy_update)
         self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd)
 
         # Add lookahead_analysis subcommand
         lookahead_analayis_cmd = subparsers.add_parser(
-            'lookahead-analysis',
+            "lookahead-analysis",
             help="Check for potential look ahead bias.",
-            parents=[_common_parser, _strategy_parser]
+            parents=[_common_parser, _strategy_parser],
         )
         lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis)
 
-        self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
-                         parser=lookahead_analayis_cmd)
+        self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, parser=lookahead_analayis_cmd)
 
         # Add recursive_analysis subcommand
         recursive_analayis_cmd = subparsers.add_parser(
-            'recursive-analysis',
+            "recursive-analysis",
             help="Check for potential recursive formula issue.",
-            parents=[_common_parser, _strategy_parser]
+            parents=[_common_parser, _strategy_parser],
         )
         recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
 
-        self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,
-                         parser=recursive_analayis_cmd)
+        self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS, parser=recursive_analayis_cmd)

freqtrade/commands/build_config_commands.py

@@ -45,7 +45,7 @@ def ask_user_overwrite(config_path: Path) -> bool:
         },
     ]
     answers = prompt(questions)
-    return answers['overwrite']
+    return answers["overwrite"]
 
 
 def ask_user_config() -> Dict[str, Any]:
@@ -65,7 +65,7 @@ def ask_user_config() -> Dict[str, Any]:
             "type": "text",
             "name": "stake_currency",
             "message": "Please insert your stake currency:",
-            "default": 'USDT',
+            "default": "USDT",
         },
         {
             "type": "text",
@@ -73,36 +73,38 @@ def ask_user_config() -> Dict[str, Any]:
             "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
             "default": "unlimited",
             "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
-            "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
-            if val == UNLIMITED_STAKE_AMOUNT
-            else val
+            "filter": lambda val: (
+                '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT else val
+            ),
         },
         {
             "type": "text",
             "name": "max_open_trades",
             "message": "Please insert max_open_trades (Integer or -1 for unlimited open trades):",
             "default": "3",
-            "validate": lambda val: validate_is_int(val)
+            "validate": lambda val: validate_is_int(val),
         },
         {
             "type": "select",
             "name": "timeframe_in_config",
             "message": "Time",
-            "choices": ["Have the strategy define timeframe.", "Override in configuration."]
+            "choices": ["Have the strategy define timeframe.", "Override in configuration."],
         },
         {
             "type": "text",
             "name": "timeframe",
             "message": "Please insert your desired timeframe (e.g. 5m):",
             "default": "5m",
-            "when": lambda x: x["timeframe_in_config"] == 'Override in configuration.'
-
+            "when": lambda x: x["timeframe_in_config"] == "Override in configuration.",
         },
         {
             "type": "text",
             "name": "fiat_display_currency",
-            "message": "Please insert your display Currency (for reporting):",
-            "default": 'USD',
+            "message": (
+                "Please insert your display Currency for reporting "
+                "(leave empty to disable FIAT conversion):"
+            ),
+            "default": "USD",
         },
         {
             "type": "select",
@@ -111,6 +113,7 @@ def ask_user_config() -> Dict[str, Any]:
             "choices": [
                 "binance",
                 "binanceus",
+                "bingx",
                 "gate",
                 "htx",
                 "kraken",
@@ -125,33 +128,33 @@ def ask_user_config() -> Dict[str, Any]:
             "name": "trading_mode",
             "message": "Do you want to trade Perpetual Swaps (perpetual futures)?",
             "default": False,
-            "filter": lambda val: 'futures' if val else 'spot',
-            "when": lambda x: x["exchange_name"] in ['binance', 'gate', 'okx'],
+            "filter": lambda val: "futures" if val else "spot",
+            "when": lambda x: x["exchange_name"] in ["binance", "gate", "okx", "bybit"],
         },
         {
             "type": "autocomplete",
             "name": "exchange_name",
             "message": "Type your exchange name (Must be supported by ccxt)",
             "choices": available_exchanges(),
-            "when": lambda x: x["exchange_name"] == 'other'
+            "when": lambda x: x["exchange_name"] == "other",
         },
         {
             "type": "password",
             "name": "exchange_key",
             "message": "Insert Exchange Key",
-            "when": lambda x: not x['dry_run']
+            "when": lambda x: not x["dry_run"],
         },
         {
             "type": "password",
             "name": "exchange_secret",
             "message": "Insert Exchange Secret",
-            "when": lambda x: not x['dry_run']
+            "when": lambda x: not x["dry_run"],
         },
         {
             "type": "password",
             "name": "exchange_key_password",
             "message": "Insert Exchange API Key password",
-            "when": lambda x: not x['dry_run'] and x['exchange_name'] in ('kucoin', 'okx')
+            "when": lambda x: not x["dry_run"] and x["exchange_name"] in ("kucoin", "okx"),
         },
         {
             "type": "confirm",
@@ -163,13 +166,13 @@ def ask_user_config() -> Dict[str, Any]:
             "type": "password",
             "name": "telegram_token",
             "message": "Insert Telegram token",
-            "when": lambda x: x['telegram']
+            "when": lambda x: x["telegram"],
         },
         {
             "type": "password",
             "name": "telegram_chat_id",
             "message": "Insert Telegram chat id",
-            "when": lambda x: x['telegram']
+            "when": lambda x: x["telegram"],
         },
         {
             "type": "confirm",
@@ -180,23 +183,25 @@ def ask_user_config() -> Dict[str, Any]:
         {
             "type": "text",
             "name": "api_server_listen_addr",
-            "message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
-                        "otherwise best left untouched)"),
-            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",
-            "when": lambda x: x['api_server']
+            "message": (
+                "Insert Api server Listen Address (0.0.0.0 for docker, "
+                "otherwise best left untouched)"
+            ),
+            "default": "127.0.0.1" if not running_in_docker() else "0.0.0.0",  # noqa: S104
+            "when": lambda x: x["api_server"],
         },
         {
             "type": "text",
             "name": "api_server_username",
             "message": "Insert api-server username",
             "default": "freqtrader",
-            "when": lambda x: x['api_server']
+            "when": lambda x: x["api_server"],
         },
         {
             "type": "password",
             "name": "api_server_password",
             "message": "Insert api-server password",
-            "when": lambda x: x['api_server']
+            "when": lambda x: x["api_server"],
         },
     ]
     answers = prompt(questions)
@@ -205,15 +210,11 @@ def ask_user_config() -> Dict[str, Any]:
         # Interrupted questionary sessions return an empty dict.
         raise OperationalException("User interrupted interactive questions.")
     # Ensure default is set for non-futures exchanges
-    answers['trading_mode'] = answers.get('trading_mode', "spot")
-    answers['margin_mode'] = (
-        'isolated'
-        if answers.get('trading_mode') == 'futures'
-        else ''
-    )
+    answers["trading_mode"] = answers.get("trading_mode", "spot")
+    answers["margin_mode"] = "isolated" if answers.get("trading_mode") == "futures" else ""
     # Force JWT token to be a random string
-    answers['api_server_jwt_key'] = secrets.token_hex()
-    answers['api_server_ws_token'] = secrets.token_urlsafe(25)
+    answers["api_server_jwt_key"] = secrets.token_hex()
+    answers["api_server_ws_token"] = secrets.token_urlsafe(25)
 
     return answers
 
@@ -225,26 +226,26 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
     :param selections: Dict containing selections taken by the user.
     """
     from jinja2.exceptions import TemplateNotFound
+
     try:
         exchange_template = MAP_EXCHANGE_CHILDCLASS.get(
-            selections['exchange_name'], selections['exchange_name'])
+            selections["exchange_name"], selections["exchange_name"]
+        )
 
-        selections['exchange'] = render_template(
-            templatefile=f"subtemplates/exchange_{exchange_template}.j2",
-            arguments=selections
+        selections["exchange"] = render_template(
+            templatefile=f"subtemplates/exchange_{exchange_template}.j2", arguments=selections
         )
     except TemplateNotFound:
-        selections['exchange'] = render_template(
-            templatefile="subtemplates/exchange_generic.j2",
-            arguments=selections
+        selections["exchange"] = render_template(
+            templatefile="subtemplates/exchange_generic.j2", arguments=selections
         )
 
-    config_text = render_template(templatefile='base_config.json.j2',
-                                  arguments=selections)
+    config_text = render_template(templatefile="base_config.json.j2", arguments=selections)
 
     logger.info(f"Writing config to `{config_path}`.")
     logger.info(
-        "Please make sure to check the configuration contents and adjust settings to your needs.")
+        "Please make sure to check the configuration contents and adjust settings to your needs."
+    )
 
     config_path.write_text(config_text)
 
@@ -255,7 +256,7 @@ def start_new_config(args: Dict[str, Any]) -> None:
     Asking the user questions to fill out the template accordingly.
     """
 
-    config_path = Path(args['config'][0])
+    config_path = Path(args["config"][0])
     chown_user_directory(config_path.parent)
     if config_path.exists():
         overwrite = ask_user_overwrite(config_path)
@@ -264,22 +265,22 @@ def start_new_config(args: Dict[str, Any]) -> None:
         else:
             raise OperationalException(
                 f"Configuration file `{config_path}` already exists. "
-                "Please delete it or use a different configuration file name.")
+                "Please delete it or use a different configuration file name."
+            )
     selections = ask_user_config()
     deploy_new_config(config_path, selections)
 
 
 def start_show_config(args: Dict[str, Any]) -> None:
-
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE, set_dry=False)
 
     # TODO: Sanitize from sensitive info before printing
 
     print("Your combined configuration is:")
     config_sanitized = sanitize_config(
-        config['original_config'],
-        show_sensitive=args.get('show_sensitive', False)
+        config["original_config"], show_sensitive=args.get("show_sensitive", False)
     )
 
     from rich import print_json
+
     print_json(data=config_sanitized)

freqtrade/commands/data_commands.py

@@ -5,14 +5,18 @@ from typing import Any, Dict
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
 from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config
-from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
-                                      convert_trades_to_ohlcv)
+from freqtrade.data.converter import (
+    convert_ohlcv_format,
+    convert_trades_format,
+    convert_trades_to_ohlcv,
+)
 from freqtrade.data.history import download_data_main
 from freqtrade.enums import CandleType, RunMode, TradingMode
 from freqtrade.exceptions import ConfigurationError
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
+from freqtrade.util import print_rich_table
 from freqtrade.util.migrations import migrate_data
 
 
@@ -20,14 +24,17 @@ logger = logging.getLogger(__name__)
 
 
 def _check_data_config_download_sanity(config: Config) -> None:
-    if 'days' in config and 'timerange' in config:
-        raise ConfigurationError("--days and --timerange are mutually exclusive. "
-                                 "You can only specify one or the other.")
+    if "days" in config and "timerange" in config:
+        raise ConfigurationError(
+            "--days and --timerange are mutually exclusive. "
+            "You can only specify one or the other."
+        )
 
-    if 'pairs' not in config:
+    if "pairs" not in config:
         raise ConfigurationError(
             "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
+            "Please check the documentation on how to configure this."
+        )
 
 
 def start_download_data(args: Dict[str, Any]) -> None:
@@ -46,38 +53,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
 
 
 def start_convert_trades(args: Dict[str, Any]) -> None:
-
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
     timerange = TimeRange()
 
     # Remove stake-currency to skip checks which are not relevant for datadownload
-    config['stake_currency'] = ''
+    config["stake_currency"] = ""
 
-    if 'timeframes' not in config:
-        config['timeframes'] = DL_DATA_TIMEFRAMES
+    if "timeframes" not in config:
+        config["timeframes"] = DL_DATA_TIMEFRAMES
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config, validate=False)
     # Manual validations of relevant settings
 
-    for timeframe in config['timeframes']:
+    for timeframe in config["timeframes"]:
         exchange.validate_timeframes(timeframe)
     available_pairs = [
-        p for p in exchange.get_markets(
-            tradable_only=True, active_only=not config.get('include_inactive')
-            ).keys()
+        p
+        for p in exchange.get_markets(
+            tradable_only=True, active_only=not config.get("include_inactive")
+        ).keys()
     ]
 
     expanded_pairs = dynamic_expand_pairlist(config, available_pairs)
 
     # Convert downloaded trade data to different timeframes
     convert_trades_to_ohlcv(
-        pairs=expanded_pairs, timeframes=config['timeframes'],
-        datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
-        data_format_ohlcv=config['dataformat_ohlcv'],
-        data_format_trades=config['dataformat_trades'],
-        candle_type=config.get('candle_type_def', CandleType.SPOT)
+        pairs=expanded_pairs,
+        timeframes=config["timeframes"],
+        datadir=config["datadir"],
+        timerange=timerange,
+        erase=bool(config.get("erase")),
+        data_format_ohlcv=config["dataformat_ohlcv"],
+        data_format_trades=config["dataformat_trades"],
+        candle_type=config.get("candle_type_def", CandleType.SPOT),
     )
 
 
@@ -88,14 +98,19 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
         migrate_data(config)
-        convert_ohlcv_format(config,
-                             convert_from=args['format_from'],
-                             convert_to=args['format_to'],
-                             erase=args['erase'])
+        convert_ohlcv_format(
+            config,
+            convert_from=args["format_from"],
+            convert_to=args["format_to"],
+            erase=args["erase"],
+        )
     else:
-        convert_trades_format(config,
-                              convert_from=args['format_from_trades'], convert_to=args['format_to'],
-                              erase=args['erase'])
+        convert_trades_format(
+            config,
+            convert_from=args["format_from_trades"],
+            convert_to=args["format_to"],
+            erase=args["erase"],
+        )
 
 
 def start_list_data(args: Dict[str, Any]) -> None:
@@ -105,48 +120,54 @@ def start_list_data(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    from tabulate import tabulate
-
     from freqtrade.data.history import get_datahandler
-    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+
+    dhc = get_datahandler(config["datadir"], config["dataformat_ohlcv"])
 
     paircombs = dhc.ohlcv_get_available_data(
-        config['datadir'],
-        config.get('trading_mode', TradingMode.SPOT)
-        )
+        config["datadir"], config.get("trading_mode", TradingMode.SPOT)
+    )
 
-    if args['pairs']:
-        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
-
-    print(f"Found {len(paircombs)} pair / timeframe combinations.")
-    if not config.get('show_timerange'):
+    if args["pairs"]:
+        paircombs = [comb for comb in paircombs if comb[0] in args["pairs"]]
+    title = f"Found {len(paircombs)} pair / timeframe combinations."
+    if not config.get("show_timerange"):
         groupedpair = defaultdict(list)
         for pair, timeframe, candle_type in sorted(
-            paircombs,
-            key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
+            paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
         ):
             groupedpair[(pair, candle_type)].append(timeframe)
 
         if groupedpair:
-            print(tabulate([
-                (pair, ', '.join(timeframes), candle_type)
-                for (pair, candle_type), timeframes in groupedpair.items()
-            ],
-                headers=("Pair", "Timeframe", "Type"),
-                tablefmt='psql', stralign='right'))
+            print_rich_table(
+                [
+                    (pair, ", ".join(timeframes), candle_type)
+                    for (pair, candle_type), timeframes in groupedpair.items()
+                ],
+                ("Pair", "Timeframe", "Type"),
+                title,
+                table_kwargs={"min_width": 50},
+            )
     else:
-        paircombs1 = [(
-            pair, timeframe, candle_type,
-            *dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
-        ) for pair, timeframe, candle_type in paircombs]
-
-        print(tabulate([
-            (pair, timeframe, candle_type,
-                start.strftime(DATETIME_PRINT_FORMAT),
-                end.strftime(DATETIME_PRINT_FORMAT), length)
-            for pair, timeframe, candle_type, start, end, length in sorted(
-                paircombs1,
-                key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
+        paircombs1 = [
+            (pair, timeframe, candle_type, *dhc.ohlcv_data_min_max(pair, timeframe, candle_type))
+            for pair, timeframe, candle_type in paircombs
+        ]
+        print_rich_table(
+            [
+                (
+                    pair,
+                    timeframe,
+                    candle_type,
+                    start.strftime(DATETIME_PRINT_FORMAT),
+                    end.strftime(DATETIME_PRINT_FORMAT),
+                    str(length),
+                )
+                for pair, timeframe, candle_type, start, end, length in sorted(
+                    paircombs1, key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
+                )
             ],
-            headers=("Pair", "Timeframe", "Type", 'From', 'To', 'Candles'),
-            tablefmt='psql', stralign='right'))
+            ("Pair", "Timeframe", "Type", "From", "To", "Candles"),
+            summary=title,
+            table_kwargs={"min_width": 50},
+        )

freqtrade/commands/db_commands.py

@@ -19,9 +19,9 @@ def start_convert_db(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    init_db(config['db_url'])
+    init_db(config["db_url"])
     session_target = Trade.session
-    init_db(config['db_url_from'])
+    init_db(config["db_url_from"])
     logger.info("Starting db migration.")
 
     trade_count = 0
@@ -47,9 +47,11 @@ def start_convert_db(args: Dict[str, Any]) -> None:
     max_order_id = session_target.scalar(select(func.max(Order.id)))
     max_pairlock_id = session_target.scalar(select(func.max(PairLock.id)))
 
-    set_sequence_ids(session_target.get_bind(),
-                     trade_id=max_trade_id,
-                     order_id=max_order_id,
-                     pairlock_id=max_pairlock_id)
+    set_sequence_ids(
+        session_target.get_bind(),
+        trade_id=max_trade_id,
+        order_id=max_order_id,
+        pairlock_id=max_pairlock_id,
+    )
 
     logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.")

freqtrade/commands/deploy_commands.py

@@ -38,7 +38,7 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
     """
     Deploy new strategy from template to strategy_path
     """
-    fallback = 'full'
+    fallback = "full"
     attributes = render_template_with_fallback(
         templatefile=f"strategy_subtemplates/strategy_attributes_{subtemplate}.j2",
         templatefallbackfile=f"strategy_subtemplates/strategy_attributes_{fallback}.j2",
@@ -64,33 +64,35 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
         templatefallbackfile="strategy_subtemplates/strategy_methods_empty.j2",
     )
 
-    strategy_text = render_template(templatefile='base_strategy.py.j2',
-                                    arguments={"strategy": strategy_name,
-                                               "attributes": attributes,
-                                               "indicators": indicators,
-                                               "buy_trend": buy_trend,
-                                               "sell_trend": sell_trend,
-                                               "plot_config": plot_config,
-                                               "additional_methods": additional_methods,
-                                               })
+    strategy_text = render_template(
+        templatefile="base_strategy.py.j2",
+        arguments={
+            "strategy": strategy_name,
+            "attributes": attributes,
+            "indicators": indicators,
+            "buy_trend": buy_trend,
+            "sell_trend": sell_trend,
+            "plot_config": plot_config,
+            "additional_methods": additional_methods,
+        },
+    )
 
     logger.info(f"Writing strategy to `{strategy_path}`.")
     strategy_path.write_text(strategy_text)
 
 
 def start_new_strategy(args: Dict[str, Any]) -> None:
-
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     if "strategy" in args and args["strategy"]:
-
-        new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
+        new_path = config["user_data_dir"] / USERPATH_STRATEGIES / (args["strategy"] + ".py")
 
         if new_path.exists():
-            raise OperationalException(f"`{new_path}` already exists. "
-                                       "Please choose another Strategy Name.")
+            raise OperationalException(
+                f"`{new_path}` already exists. Please choose another Strategy Name."
+            )
 
-        deploy_new_strategy(args['strategy'], new_path, args['template'])
+        deploy_new_strategy(args["strategy"], new_path, args["template"])
 
     else:
         raise ConfigurationError("`new-strategy` requires --strategy to be set.")
@@ -100,8 +102,8 @@ def clean_ui_subdir(directory: Path):
     if directory.is_dir():
         logger.info("Removing UI directory content.")
 
-        for p in reversed(list(directory.glob('**/*'))):  # iterate contents from leaves to root
-            if p.name in ('.gitkeep', 'fallback_file.html'):
+        for p in reversed(list(directory.glob("**/*"))):  # iterate contents from leaves to root
+            if p.name in (".gitkeep", "fallback_file.html"):
                 continue
             if p.is_file():
                 p.unlink()
@@ -110,11 +112,11 @@ def clean_ui_subdir(directory: Path):
 
 
 def read_ui_version(dest_folder: Path) -> Optional[str]:
-    file = dest_folder / '.uiversion'
+    file = dest_folder / ".uiversion"
     if not file.is_file():
         return None
 
-    with file.open('r') as f:
+    with file.open("r") as f:
         return f.read()
 
 
@@ -133,12 +135,12 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str):
                     destfile.mkdir(exist_ok=True)
                 else:
                     destfile.write_bytes(x.read())
-    with (dest_folder / '.uiversion').open('w') as f:
+    with (dest_folder / ".uiversion").open("w") as f:
         f.write(version)
 
 
 def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]:
-    base_url = 'https://api.github.com/repos/freqtrade/frequi/'
+    base_url = "https://api.github.com/repos/freqtrade/frequi/"
     # Get base UI Repo path
 
     resp = requests.get(f"{base_url}releases", timeout=req_timeout)
@@ -146,42 +148,41 @@ def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]:
     r = resp.json()
 
     if version:
-        tmp = [x for x in r if x['name'] == version]
+        tmp = [x for x in r if x["name"] == version]
         if tmp:
-            latest_version = tmp[0]['name']
-            assets = tmp[0].get('assets', [])
+            latest_version = tmp[0]["name"]
+            assets = tmp[0].get("assets", [])
         else:
             raise ValueError("UI-Version not found.")
     else:
-        latest_version = r[0]['name']
-        assets = r[0].get('assets', [])
-    dl_url = ''
+        latest_version = r[0]["name"]
+        assets = r[0].get("assets", [])
+    dl_url = ""
     if assets and len(assets) > 0:
-        dl_url = assets[0]['browser_download_url']
+        dl_url = assets[0]["browser_download_url"]
 
     # URL not found - try assets url
     if not dl_url:
-        assets = r[0]['assets_url']
+        assets = r[0]["assets_url"]
         resp = requests.get(assets, timeout=req_timeout)
         r = resp.json()
-        dl_url = r[0]['browser_download_url']
+        dl_url = r[0]["browser_download_url"]
 
     return dl_url, latest_version
 
 
 def start_install_ui(args: Dict[str, Any]) -> None:
-
-    dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/'
+    dest_folder = Path(__file__).parents[1] / "rpc/api_server/ui/installed/"
     # First make sure the assets are removed.
-    dl_url, latest_version = get_ui_download_url(args.get('ui_version'))
+    dl_url, latest_version = get_ui_download_url(args.get("ui_version"))
 
     curr_version = read_ui_version(dest_folder)
-    if curr_version == latest_version and not args.get('erase_ui_only'):
+    if curr_version == latest_version and not args.get("erase_ui_only"):
         logger.info(f"UI already up-to-date, FreqUI Version {curr_version}.")
         return
 
     clean_ui_subdir(dest_folder)
-    if args.get('erase_ui_only'):
+    if args.get("erase_ui_only"):
         logger.info("Erased UI directory content. Not downloading new version.")
     else:
         # Download a new version

freqtrade/commands/hyperopt_commands.py

@@ -2,8 +2,6 @@ import logging
 from operator import itemgetter
 from typing import Any, Dict
 
-from colorama import init as colorama_init
-
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.data.btanalysis import get_latest_hyperopt_file
 from freqtrade.enums import RunMode
@@ -18,43 +16,45 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
     """
     List hyperopt epochs previously evaluated
     """
+    from freqtrade.optimize.hyperopt_output import HyperoptOutput
     from freqtrade.optimize.hyperopt_tools import HyperoptTools
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    print_colorized = config.get('print_colorized', False)
-    print_json = config.get('print_json', False)
-    export_csv = config.get('export_csv')
-    no_details = config.get('hyperopt_list_no_details', False)
+    print_colorized = config.get("print_colorized", False)
+    print_json = config.get("print_json", False)
+    export_csv = config.get("export_csv")
+    no_details = config.get("hyperopt_list_no_details", False)
     no_header = False
 
     results_file = get_latest_hyperopt_file(
-        config['user_data_dir'] / 'hyperopt_results',
-        config.get('hyperoptexportfilename'))
+        config["user_data_dir"] / "hyperopt_results", config.get("hyperoptexportfilename")
+    )
 
     # Previous evaluations
     epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
 
-    if print_colorized:
-        colorama_init(autoreset=True)
-
     if not export_csv:
         try:
-            print(HyperoptTools.get_result_table(config, epochs, total_epochs,
-                                                 not config.get('hyperopt_list_best', False),
-                                                 print_colorized, 0))
+            h_out = HyperoptOutput()
+            h_out.add_data(
+                config,
+                epochs,
+                total_epochs,
+                not config.get("hyperopt_list_best", False),
+            )
+            h_out.print(print_colorized=print_colorized)
+
         except KeyboardInterrupt:
-            print('User interrupted..')
+            print("User interrupted..")
 
     if epochs and not no_details:
-        sorted_epochs = sorted(epochs, key=itemgetter('loss'))
+        sorted_epochs = sorted(epochs, key=itemgetter("loss"))
         results = sorted_epochs[0]
         HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header)
 
     if epochs and export_csv:
-        HyperoptTools.export_csv_file(
-            config, epochs, export_csv
-        )
+        HyperoptTools.export_csv_file(config, epochs, export_csv)
 
 
 def start_hyperopt_show(args: Dict[str, Any]) -> None:
@@ -65,13 +65,13 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    print_json = config.get('print_json', False)
-    no_header = config.get('hyperopt_show_no_header', False)
+    print_json = config.get("print_json", False)
+    no_header = config.get("hyperopt_show_no_header", False)
     results_file = get_latest_hyperopt_file(
-        config['user_data_dir'] / 'hyperopt_results',
-        config.get('hyperoptexportfilename'))
+        config["user_data_dir"] / "hyperopt_results", config.get("hyperoptexportfilename")
+    )
 
-    n = config.get('hyperopt_show_index', -1)
+    n = config.get("hyperopt_show_index", -1)
 
     # Previous evaluations
     epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
@@ -80,10 +80,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
 
     if n > filtered_epochs:
         raise OperationalException(
-            f"The index of the epoch to show should be less than {filtered_epochs + 1}.")
+            f"The index of the epoch to show should be less than {filtered_epochs + 1}."
+        )
     if n < -filtered_epochs:
         raise OperationalException(
-            f"The index of the epoch to show should be greater than {-filtered_epochs - 1}.")
+            f"The index of the epoch to show should be greater than {-filtered_epochs - 1}."
+        )
 
     # Translate epoch index from human-readable format to pythonic
     if n > 0:
@@ -92,13 +94,18 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
     if epochs:
         val = epochs[n]
 
-        metrics = val['results_metrics']
-        if 'strategy_name' in metrics:
-            strategy_name = metrics['strategy_name']
-            show_backtest_result(strategy_name, metrics,
-                                 metrics['stake_currency'], config.get('backtest_breakdown', []))
+        metrics = val["results_metrics"]
+        if "strategy_name" in metrics:
+            strategy_name = metrics["strategy_name"]
+            show_backtest_result(
+                strategy_name,
+                metrics,
+                metrics["stake_currency"],
+                config.get("backtest_breakdown", []),
+            )
 
             HyperoptTools.try_export_params(config, strategy_name, val)
 
-        HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
-                                         header_str="Epoch details")
+        HyperoptTools.show_epoch_details(
+            val, total_epochs, print_json, no_header, header_str="Epoch details"
+        )

freqtrade/commands/list_commands.py

@@ -4,9 +4,9 @@ import sys
 from typing import Any, Dict, List, Union
 
 import rapidjson
-from colorama import Fore, Style
-from colorama import init as colorama_init
-from tabulate import tabulate
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
 
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
@@ -14,7 +14,8 @@ from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.exchange import list_available_exchanges, market_is_active
 from freqtrade.misc import parse_db_uri_for_logging, plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.types import ValidExchangesType
+from freqtrade.types.valid_exchanges_type import ValidExchangesType
+from freqtrade.util import print_rich_table
 
 
 logger = logging.getLogger(__name__)
@@ -26,71 +27,95 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    exchanges = list_available_exchanges(args['list_exchanges_all'])
+    available_exchanges: List[ValidExchangesType] = list_available_exchanges(
+        args["list_exchanges_all"]
+    )
 
-    if args['print_one_column']:
-        print('\n'.join([e['name'] for e in exchanges]))
+    if args["print_one_column"]:
+        print("\n".join([e["name"] for e in available_exchanges]))
     else:
-        headers = {
-            'name': 'Exchange name',
-            'supported': 'Supported',
-            'trade_modes': 'Markets',
-            'comment': 'Reason',
-            }
-        headers.update({'valid': 'Valid'} if args['list_exchanges_all'] else {})
-
-        def build_entry(exchange: ValidExchangesType, valid: bool):
-            valid_entry = {'valid': exchange['valid']} if valid else {}
-            result: Dict[str, Union[str, bool]] = {
-                'name': exchange['name'],
-                **valid_entry,
-                'supported': 'Official' if exchange['supported'] else '',
-                'trade_modes': ', '.join(
-                    (f"{a['margin_mode']} " if a['margin_mode'] else '') + a['trading_mode']
-                    for a in exchange['trade_modes']
-                ),
-                'comment': exchange['comment'],
-            }
-
-            return result
-
-        if args['list_exchanges_all']:
-            print("All exchanges supported by the ccxt library:")
-            exchanges = [build_entry(e, True) for e in exchanges]
+        if args["list_exchanges_all"]:
+            title = (
+                f"All exchanges supported by the ccxt library "
+                f"({len(available_exchanges)} exchanges):"
+            )
         else:
-            print("Exchanges available for Freqtrade:")
-            exchanges = [build_entry(e, False) for e in exchanges if e['valid'] is not False]
+            available_exchanges = [e for e in available_exchanges if e["valid"] is not False]
+            title = f"Exchanges available for Freqtrade ({len(available_exchanges)} exchanges):"
 
-        print(tabulate(exchanges, headers=headers, ))
+        table = Table(title=title)
+
+        table.add_column("Exchange Name")
+        table.add_column("Markets")
+        table.add_column("Reason")
+
+        for exchange in available_exchanges:
+            name = Text(exchange["name"])
+            if exchange["supported"]:
+                name.append(" (Official)", style="italic")
+                name.stylize("green bold")
+
+            trade_modes = Text(
+                ", ".join(
+                    (f"{a.get('margin_mode', '')} {a['trading_mode']}").lstrip()
+                    for a in exchange["trade_modes"]
+                ),
+                style="",
+            )
+            if exchange["dex"]:
+                trade_modes = Text("DEX: ") + trade_modes
+                trade_modes.stylize("bold", 0, 3)
+
+            table.add_row(
+                name,
+                trade_modes,
+                exchange["comment"],
+                style=None if exchange["valid"] else "red",
+            )
+            # table.add_row(*[exchange[header] for header in headers])
+
+        console = Console()
+        console.print(table)
 
 
 def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
-    if print_colorized:
-        colorama_init(autoreset=True)
-        red = Fore.RED
-        yellow = Fore.YELLOW
-        reset = Style.RESET_ALL
-    else:
-        red = ''
-        yellow = ''
-        reset = ''
-
-    names = [s['name'] for s in objs]
-    objs_to_print = [{
-        'name': s['name'] if s['name'] else "--",
-        'location': s['location_rel'],
-        'status': (red + "LOAD FAILED" + reset if s['class'] is None
-                   else "OK" if names.count(s['name']) == 1
-                   else yellow + "DUPLICATE NAME" + reset)
-    } for s in objs]
+    names = [s["name"] for s in objs]
+    objs_to_print: List[Dict[str, Union[Text, str]]] = [
+        {
+            "name": Text(s["name"] if s["name"] else "--"),
+            "location": s["location_rel"],
+            "status": (
+                Text("LOAD FAILED", style="bold red")
+                if s["class"] is None
+                else Text("OK", style="bold green")
+                if names.count(s["name"]) == 1
+                else Text("DUPLICATE NAME", style="bold yellow")
+            ),
+        }
+        for s in objs
+    ]
     for idx, s in enumerate(objs):
-        if 'hyperoptable' in s:
-            objs_to_print[idx].update({
-                'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No",
-                'buy-Params': len(s['hyperoptable'].get('buy', [])),
-                'sell-Params': len(s['hyperoptable'].get('sell', [])),
-            })
-    print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right'))
+        if "hyperoptable" in s:
+            objs_to_print[idx].update(
+                {
+                    "hyperoptable": "Yes" if s["hyperoptable"]["count"] > 0 else "No",
+                    "buy-Params": str(len(s["hyperoptable"].get("buy", []))),
+                    "sell-Params": str(len(s["hyperoptable"].get("sell", []))),
+                }
+            )
+    table = Table()
+
+    for header in objs_to_print[0].keys():
+        table.add_column(header.capitalize(), justify="right")
+
+    for row in objs_to_print:
+        table.add_row(*[row[header] for header in objs_to_print[0].keys()])
+
+    console = Console(
+        color_system="auto" if print_colorized else None,
+        width=200 if "pytest" in sys.modules else None,
+    )
+    console.print(table)
 
 
 def start_list_strategies(args: Dict[str, Any]) -> None:
@@ -100,19 +125,20 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     strategy_objs = StrategyResolver.search_all_objects(
-        config, not args['print_one_column'], config.get('recursive_strategy_search', False))
+        config, not args["print_one_column"], config.get("recursive_strategy_search", False)
+    )
     # Sort alphabetically
-    strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
+    strategy_objs = sorted(strategy_objs, key=lambda x: x["name"])
     for obj in strategy_objs:
-        if obj['class']:
-            obj['hyperoptable'] = obj['class'].detect_all_parameters()
+        if obj["class"]:
+            obj["hyperoptable"] = obj["class"].detect_all_parameters()
         else:
-            obj['hyperoptable'] = {'count': 0}
+            obj["hyperoptable"] = {"count": 0}
 
-    if args['print_one_column']:
-        print('\n'.join([s['name'] for s in strategy_objs]))
+    if args["print_one_column"]:
+        print("\n".join([s["name"] for s in strategy_objs]))
     else:
-        _print_objs_tabular(strategy_objs, config.get('print_colorized', False))
+        _print_objs_tabular(strategy_objs, config.get("print_colorized", False))
 
 
 def start_list_freqAI_models(args: Dict[str, Any]) -> None:
@@ -121,13 +147,14 @@ def start_list_freqAI_models(args: Dict[str, Any]) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
-    model_objs = FreqaiModelResolver.search_all_objects(config, not args['print_one_column'])
+
+    model_objs = FreqaiModelResolver.search_all_objects(config, not args["print_one_column"])
     # Sort alphabetically
-    model_objs = sorted(model_objs, key=lambda x: x['name'])
-    if args['print_one_column']:
-        print('\n'.join([s['name'] for s in model_objs]))
+    model_objs = sorted(model_objs, key=lambda x: x["name"])
+    if args["print_one_column"]:
+        print("\n".join([s["name"] for s in model_objs]))
     else:
-        _print_objs_tabular(model_objs, config.get('print_colorized', False))
+        _print_objs_tabular(model_objs, config.get("print_colorized", False))
 
 
 def start_list_timeframes(args: Dict[str, Any]) -> None:
@@ -136,16 +163,18 @@ def start_list_timeframes(args: Dict[str, Any]) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
     # Do not use timeframe set in the config
-    config['timeframe'] = None
+    config["timeframe"] = None
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config, validate=False)
 
-    if args['print_one_column']:
-        print('\n'.join(exchange.timeframes))
+    if args["print_one_column"]:
+        print("\n".join(exchange.timeframes))
     else:
-        print(f"Timeframes available for the exchange `{exchange.name}`: "
-              f"{', '.join(exchange.timeframes)}")
+        print(
+            f"Timeframes available for the exchange `{exchange.name}`: "
+            f"{', '.join(exchange.timeframes)}"
+        )
 
 
 def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
@@ -161,51 +190,75 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
     exchange = ExchangeResolver.load_exchange(config, validate=False)
 
     # By default only active pairs/markets are to be shown
-    active_only = not args.get('list_pairs_all', False)
+    active_only = not args.get("list_pairs_all", False)
 
-    base_currencies = args.get('base_currencies', [])
-    quote_currencies = args.get('quote_currencies', [])
+    base_currencies = args.get("base_currencies", [])
+    quote_currencies = args.get("quote_currencies", [])
 
     try:
-        pairs = exchange.get_markets(base_currencies=base_currencies,
-                                     quote_currencies=quote_currencies,
-                                     tradable_only=pairs_only,
-                                     active_only=active_only)
+        pairs = exchange.get_markets(
+            base_currencies=base_currencies,
+            quote_currencies=quote_currencies,
+            tradable_only=pairs_only,
+            active_only=active_only,
+        )
         # Sort the pairs/markets by symbol
         pairs = dict(sorted(pairs.items()))
     except Exception as e:
         raise OperationalException(f"Cannot get markets. Reason: {e}") from e
 
     else:
-        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
-                       ("active " if active_only else "") +
-                       (plural(len(pairs), "pair" if pairs_only else "market")) +
-                       (f" with {', '.join(base_currencies)} as base "
-                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
-                        if base_currencies else "") +
-                       (" and" if base_currencies and quote_currencies else "") +
-                       (f" with {', '.join(quote_currencies)} as quote "
-                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
-                        if quote_currencies else ""))
+        summary_str = (
+            (f"Exchange {exchange.name} has {len(pairs)} ")
+            + ("active " if active_only else "")
+            + (plural(len(pairs), "pair" if pairs_only else "market"))
+            + (
+                f" with {', '.join(base_currencies)} as base "
+                f"{plural(len(base_currencies), 'currency', 'currencies')}"
+                if base_currencies
+                else ""
+            )
+            + (" and" if base_currencies and quote_currencies else "")
+            + (
+                f" with {', '.join(quote_currencies)} as quote "
+                f"{plural(len(quote_currencies), 'currency', 'currencies')}"
+                if quote_currencies
+                else ""
+            )
+        )
 
-        headers = ["Id", "Symbol", "Base", "Quote", "Active",
-                   "Spot", "Margin", "Future", "Leverage"]
+        headers = [
+            "Id",
+            "Symbol",
+            "Base",
+            "Quote",
+            "Active",
+            "Spot",
+            "Margin",
+            "Future",
+            "Leverage",
+        ]
 
-        tabular_data = [{
-                'Id': v['id'],
-                'Symbol': v['symbol'],
-                'Base': v['base'],
-                'Quote': v['quote'],
-                'Active': market_is_active(v),
-                'Spot': 'Spot' if exchange.market_is_spot(v) else '',
-                'Margin': 'Margin' if exchange.market_is_margin(v) else '',
-                'Future': 'Future' if exchange.market_is_future(v) else '',
-                'Leverage': exchange.get_max_leverage(v['symbol'], 20)
-            } for _, v in pairs.items()]
+        tabular_data = [
+            {
+                "Id": v["id"],
+                "Symbol": v["symbol"],
+                "Base": v["base"],
+                "Quote": v["quote"],
+                "Active": market_is_active(v),
+                "Spot": "Spot" if exchange.market_is_spot(v) else "",
+                "Margin": "Margin" if exchange.market_is_margin(v) else "",
+                "Future": "Future" if exchange.market_is_future(v) else "",
+                "Leverage": exchange.get_max_leverage(v["symbol"], 20),
+            }
+            for _, v in pairs.items()
+        ]
 
-        if (args.get('print_one_column', False) or
-                args.get('list_pairs_print_json', False) or
-                args.get('print_csv', False)):
+        if (
+            args.get("print_one_column", False)
+            or args.get("list_pairs_print_json", False)
+            or args.get("print_csv", False)
+        ):
             # Print summary string in the log in case of machine-readable
             # regular formats.
             logger.info(f"{summary_str}.")
@@ -215,24 +268,24 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
             print()
 
         if pairs:
-            if args.get('print_list', False):
+            if args.get("print_list", False):
                 # print data as a list, with human-readable summary
                 print(f"{summary_str}: {', '.join(pairs.keys())}.")
-            elif args.get('print_one_column', False):
-                print('\n'.join(pairs.keys()))
-            elif args.get('list_pairs_print_json', False):
+            elif args.get("print_one_column", False):
+                print("\n".join(pairs.keys()))
+            elif args.get("list_pairs_print_json", False):
                 print(rapidjson.dumps(list(pairs.keys()), default=str))
-            elif args.get('print_csv', False):
+            elif args.get("print_csv", False):
                 writer = csv.DictWriter(sys.stdout, fieldnames=headers)
                 writer.writeheader()
                 writer.writerows(tabular_data)
             else:
-                # print data as a table, with the human-readable summary
-                print(f"{summary_str}:")
-                print(tabulate(tabular_data, headers='keys', tablefmt='psql', stralign='right'))
-        elif not (args.get('print_one_column', False) or
-                  args.get('list_pairs_print_json', False) or
-                  args.get('print_csv', False)):
+                print_rich_table(tabular_data, headers, summary_str)
+        elif not (
+            args.get("print_one_column", False)
+            or args.get("list_pairs_print_json", False)
+            or args.get("print_csv", False)
+        ):
             print(f"{summary_str}.")
 
 
@@ -243,21 +296,22 @@ def start_show_trades(args: Dict[str, Any]) -> None:
     import json
 
     from freqtrade.persistence import Trade, init_db
+
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
-    if 'db_url' not in config:
+    if "db_url" not in config:
         raise ConfigurationError("--db-url is required for this command.")
 
     logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
-    init_db(config['db_url'])
+    init_db(config["db_url"])
     tfilter = []
 
-    if config.get('trade_ids'):
-        tfilter.append(Trade.id.in_(config['trade_ids']))
+    if config.get("trade_ids"):
+        tfilter.append(Trade.id.in_(config["trade_ids"]))
 
     trades = Trade.get_trades(tfilter).all()
     logger.info(f"Printing {len(trades)} Trades: ")
-    if config.get('print_json', False):
+    if config.get("print_json", False):
         print(json.dumps([trade.to_json() for trade in trades], indent=4))
     else:
         for trade in trades:

freqtrade/commands/optimize_commands.py

@@ -21,20 +21,22 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
     config = setup_utils_configuration(args, method)
 
     no_unlimited_runmodes = {
-        RunMode.BACKTEST: 'backtesting',
-        RunMode.HYPEROPT: 'hyperoptimization',
+        RunMode.BACKTEST: "backtesting",
+        RunMode.HYPEROPT: "hyperoptimization",
     }
     if method in no_unlimited_runmodes.keys():
-        wallet_size = config['dry_run_wallet'] * config['tradable_balance_ratio']
+        wallet_size = config["dry_run_wallet"] * config["tradable_balance_ratio"]
         # tradable_balance_ratio
-        if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT
-                and config['stake_amount'] > wallet_size):
-            wallet = fmt_coin(wallet_size, config['stake_currency'])
-            stake = fmt_coin(config['stake_amount'], config['stake_currency'])
+        if (
+            config["stake_amount"] != constants.UNLIMITED_STAKE_AMOUNT
+            and config["stake_amount"] > wallet_size
+        ):
+            wallet = fmt_coin(wallet_size, config["stake_currency"])
+            stake = fmt_coin(config["stake_amount"], config["stake_currency"])
             raise ConfigurationError(
                 f"Starting balance ({wallet}) is smaller than stake_amount {stake}. "
                 f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`."
-                )
+            )
 
     return config
 
@@ -51,7 +53,7 @@ def start_backtesting(args: Dict[str, Any]) -> None:
     # Initialize configuration
     config = setup_optimize_configuration(args, RunMode.BACKTEST)
 
-    logger.info('Starting freqtrade in Backtesting mode')
+    logger.info("Starting freqtrade in Backtesting mode")
 
     # Initialize backtesting object
     backtesting = Backtesting(config)
@@ -68,7 +70,7 @@ def start_backtesting_show(args: Dict[str, Any]) -> None:
     from freqtrade.data.btanalysis import load_backtest_stats
     from freqtrade.optimize.optimize_reports import show_backtest_results, show_sorted_pairlist
 
-    results = load_backtest_stats(config['exportfilename'])
+    results = load_backtest_stats(config["exportfilename"])
 
     show_backtest_results(config, results)
     show_sorted_pairlist(config, results)
@@ -87,20 +89,20 @@ def start_hyperopt(args: Dict[str, Any]) -> None:
         from freqtrade.optimize.hyperopt import Hyperopt
     except ImportError as e:
         raise OperationalException(
-            f"{e}. Please ensure that the hyperopt dependencies are installed.") from e
+            f"{e}. Please ensure that the hyperopt dependencies are installed."
+        ) from e
     # Initialize configuration
     config = setup_optimize_configuration(args, RunMode.HYPEROPT)
 
-    logger.info('Starting freqtrade in Hyperopt mode')
+    logger.info("Starting freqtrade in Hyperopt mode")
 
     lock = FileLock(Hyperopt.get_lock_filename(config))
 
     try:
         with lock.acquire(timeout=1):
-
             # Remove noisy log messages
-            logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
-            logging.getLogger('filelock').setLevel(logging.WARNING)
+            logging.getLogger("hyperopt.tpe").setLevel(logging.WARNING)
+            logging.getLogger("filelock").setLevel(logging.WARNING)
 
             # Initialize backtesting object
             hyperopt = Hyperopt(config)
@@ -108,9 +110,11 @@ def start_hyperopt(args: Dict[str, Any]) -> None:
 
     except Timeout:
         logger.info("Another running instance of freqtrade Hyperopt detected.")
-        logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. "
-                    "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
-                    "or on separate machines.")
+        logger.info(
+            "Simultaneous execution of multiple Hyperopt commands is not supported. "
+            "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
+            "or on separate machines."
+        )
         logger.info("Quitting now.")
         # TODO: return False here in order to help freqtrade to exit
         # with non-zero exit code...
@@ -127,7 +131,7 @@ def start_edge(args: Dict[str, Any]) -> None:
 
     # Initialize configuration
     config = setup_optimize_configuration(args, RunMode.EDGE)
-    logger.info('Starting freqtrade in Edge mode')
+    logger.info("Starting freqtrade in Edge mode")
 
     # Initialize Edge object
     edge_cli = EdgeCli(config)

freqtrade/commands/pairlist_commands.py

@@ -17,28 +17,29 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     from freqtrade.persistence import FtNoDBContext
     from freqtrade.plugins.pairlistmanager import PairListManager
+
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
     exchange = ExchangeResolver.load_exchange(config, validate=False)
 
-    quote_currencies = args.get('quote_currencies')
+    quote_currencies = args.get("quote_currencies")
     if not quote_currencies:
-        quote_currencies = [config.get('stake_currency')]
+        quote_currencies = [config.get("stake_currency")]
     results = {}
     with FtNoDBContext():
         for curr in quote_currencies:
-            config['stake_currency'] = curr
+            config["stake_currency"] = curr
             pairlists = PairListManager(exchange, config)
             pairlists.refresh_pairlist()
             results[curr] = pairlists.whitelist
 
     for curr, pairlist in results.items():
-        if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):
+        if not args.get("print_one_column", False) and not args.get("list_pairs_print_json", False):
             print(f"Pairs for {curr}: ")
 
-        if args.get('print_one_column', False):
-            print('\n'.join(pairlist))
-        elif args.get('list_pairs_print_json', False):
+        if args.get("print_one_column", False):
+            print("\n".join(pairlist))
+        elif args.get("list_pairs_print_json", False):
             print(rapidjson.dumps(list(pairlist), default=str))
         else:
             print(pairlist)

freqtrade/commands/plot_commands.py

@@ -6,10 +6,11 @@ from freqtrade.exceptions import ConfigurationError
 
 
 def validate_plot_args(args: Dict[str, Any]) -> None:
-    if not args.get('datadir') and not args.get('config'):
+    if not args.get("datadir") and not args.get("config"):
         raise ConfigurationError(
             "You need to specify either `--datadir` or `--config` "
-            "for plot-profit and plot-dataframe.")
+            "for plot-profit and plot-dataframe."
+        )
 
 
 def start_plot_dataframe(args: Dict[str, Any]) -> None:
@@ -18,6 +19,7 @@ def start_plot_dataframe(args: Dict[str, Any]) -> None:
     """
     # Import here to avoid errors if plot-dependencies are not installed.
     from freqtrade.plot.plotting import load_and_plot_trades
+
     validate_plot_args(args)
     config = setup_utils_configuration(args, RunMode.PLOT)
 
@@ -30,6 +32,7 @@ def start_plot_profit(args: Dict[str, Any]) -> None:
     """
     # Import here to avoid errors if plot-dependencies are not installed.
     from freqtrade.plot.plotting import plot_profit
+
     validate_plot_args(args)
     config = setup_utils_configuration(args, RunMode.PLOT)
 

freqtrade/commands/strategy_utils_commands.py

@@ -26,13 +26,15 @@ def start_strategy_update(args: Dict[str, Any]) -> None:
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
 
     strategy_objs = StrategyResolver.search_all_objects(
-        config, enum_failed=False, recursive=config.get('recursive_strategy_search', False))
+        config, enum_failed=False, recursive=config.get("recursive_strategy_search", False)
+    )
 
     filtered_strategy_objs = []
-    if args['strategy_list']:
+    if args["strategy_list"]:
         filtered_strategy_objs = [
-            strategy_obj for strategy_obj in strategy_objs
-            if strategy_obj['name'] in args['strategy_list']
+            strategy_obj
+            for strategy_obj in strategy_objs
+            if strategy_obj["name"] in args["strategy_list"]
         ]
 
     else:
@@ -41,8 +43,8 @@ def start_strategy_update(args: Dict[str, Any]) -> None:
 
     processed_locations = set()
     for strategy_obj in filtered_strategy_objs:
-        if strategy_obj['location'] not in processed_locations:
-            processed_locations.add(strategy_obj['location'])
+        if strategy_obj["location"] not in processed_locations:
+            processed_locations.add(strategy_obj["location"])
             start_conversion(strategy_obj, config)
 
 

freqtrade/configuration/config_secrets.py

@@ -14,9 +14,13 @@ def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
         return config
     keys_to_remove = [
         "exchange.key",
+        "exchange.apiKey",
         "exchange.secret",
         "exchange.password",
         "exchange.uid",
+        "exchange.accountId",
+        "exchange.walletAddress",
+        "exchange.privateKey",
         "telegram.token",
         "telegram.chat_id",
         "discord.webhook_url",
@@ -24,13 +28,13 @@ def sanitize_config(config: Config, *, show_sensitive: bool = False) -> Config:
     ]
     config = deepcopy(config)
     for key in keys_to_remove:
-        if '.' in key:
-            nested_keys = key.split('.')
+        if "." in key:
+            nested_keys = key.split(".")
             nested_config = config
             for nested_key in nested_keys[:-1]:
                 nested_config = nested_config.get(nested_key, {})
-            nested_config[nested_keys[-1]] = 'REDACTED'
+            nested_config[nested_keys[-1]] = "REDACTED"
         else:
-            config[key] = 'REDACTED'
+            config[key] = "REDACTED"
 
     return config

freqtrade/configuration/config_setup.py

@@ -11,7 +11,8 @@ logger = logging.getLogger(__name__)
 
 
 def setup_utils_configuration(
-        args: Dict[str, Any], method: RunMode, *, set_dry: bool = True) -> Dict[str, Any]:
+    args: Dict[str, Any], method: RunMode, *, set_dry: bool = True
+) -> Dict[str, Any]:
     """
     Prepare the configuration for utils subcommands
     :param args: Cli args from Arguments()
@@ -23,7 +24,7 @@ def setup_utils_configuration(
 
     # Ensure these modes are using Dry-run
     if set_dry:
-        config['dry_run'] = True
+        config["dry_run"] = True
     validate_config_consistency(config, preliminary=True)
 
     return config

freqtrade/configuration/config_validation.py

@@ -6,8 +6,16 @@ from typing import Any, Dict
 from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
 
-from freqtrade import constants
+from freqtrade.configuration.config_schema import (
+    CONF_SCHEMA,
+    SCHEMA_BACKTEST_REQUIRED,
+    SCHEMA_BACKTEST_REQUIRED_FINAL,
+    SCHEMA_MINIMAL_REQUIRED,
+    SCHEMA_MINIMAL_WEBSERVER,
+    SCHEMA_TRADE_REQUIRED,
+)
 from freqtrade.configuration.deprecated_settings import process_deprecated_setting
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import ConfigurationError
 
@@ -20,18 +28,16 @@ def _extend_validator(validator_class):
     Extended validator for the Freqtrade configuration JSON Schema.
     Currently it only handles defaults for subschemas.
     """
-    validate_properties = validator_class.VALIDATORS['properties']
+    validate_properties = validator_class.VALIDATORS["properties"]
 
     def set_defaults(validator, properties, instance, schema):
         for prop, subschema in properties.items():
-            if 'default' in subschema:
-                instance.setdefault(prop, subschema['default'])
+            if "default" in subschema:
+                instance.setdefault(prop, subschema["default"])
 
         yield from validate_properties(validator, properties, instance, schema)
 
-    return validators.extend(
-        validator_class, {'properties': set_defaults}
-    )
+    return validators.extend(validator_class, {"properties": set_defaults})
 
 
 FreqtradeValidator = _extend_validator(Draft4Validator)
@@ -43,28 +49,24 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
     :param conf: Config in JSON format
     :return: Returns the config if valid, otherwise throw an exception
     """
-    conf_schema = deepcopy(constants.CONF_SCHEMA)
-    if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
-        conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED
-    elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
+    conf_schema = deepcopy(CONF_SCHEMA)
+    if conf.get("runmode", RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
+        conf_schema["required"] = SCHEMA_TRADE_REQUIRED
+    elif conf.get("runmode", RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
         if preliminary:
-            conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED
         else:
-            conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
-    elif conf.get('runmode', RunMode.OTHER) == RunMode.WEBSERVER:
-        conf_schema['required'] = constants.SCHEMA_MINIMAL_WEBSERVER
+            conf_schema["required"] = SCHEMA_BACKTEST_REQUIRED_FINAL
+    elif conf.get("runmode", RunMode.OTHER) == RunMode.WEBSERVER:
+        conf_schema["required"] = SCHEMA_MINIMAL_WEBSERVER
     else:
-        conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED
+        conf_schema["required"] = SCHEMA_MINIMAL_REQUIRED
     try:
         FreqtradeValidator(conf_schema).validate(conf)
         return conf
     except ValidationError as e:
-        logger.critical(
-            f"Invalid configuration. Reason: {e}"
-        )
-        raise ValidationError(
-            best_match(Draft4Validator(conf_schema).iter_errors(conf)).message
-        )
+        logger.critical(f"Invalid configuration. Reason: {e}")
+        raise ValidationError(best_match(Draft4Validator(conf_schema).iter_errors(conf)).message)
 
 
 def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = False) -> None:
@@ -89,9 +91,10 @@ def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = Fal
     _validate_freqai_include_timeframes(conf, preliminary=preliminary)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
+    _validate_orderflow(conf)
 
     # validate configuration before returning
-    logger.info('Validating configuration ...')
+    logger.info("Validating configuration ...")
     validate_config_schema(conf, preliminary=preliminary)
 
 
@@ -100,9 +103,11 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
     If edge is disabled, either max_open_trades or stake_amount need to be set.
     :raise: ConfigurationError if config validation failed
     """
-    if (not conf.get('edge', {}).get('enabled')
-        and conf.get('max_open_trades') == float('inf')
-            and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT):
+    if (
+        not conf.get("edge", {}).get("enabled")
+        and conf.get("max_open_trades") == float("inf")
+        and conf.get("stake_amount") == UNLIMITED_STAKE_AMOUNT
+    ):
         raise ConfigurationError("`max_open_trades` and `stake_amount` cannot both be unlimited.")
 
 
@@ -111,45 +116,47 @@ def _validate_price_config(conf: Dict[str, Any]) -> None:
     When using market orders, price sides must be using the "other" side of the price
     """
     # TODO: The below could be an enforced setting when using market orders
-    if (conf.get('order_types', {}).get('entry') == 'market'
-            and conf.get('entry_pricing', {}).get('price_side') not in ('ask', 'other')):
-        raise ConfigurationError(
-            'Market entry orders require entry_pricing.price_side = "other".')
+    if conf.get("order_types", {}).get("entry") == "market" and conf.get("entry_pricing", {}).get(
+        "price_side"
+    ) not in ("ask", "other"):
+        raise ConfigurationError('Market entry orders require entry_pricing.price_side = "other".')
 
-    if (conf.get('order_types', {}).get('exit') == 'market'
-            and conf.get('exit_pricing', {}).get('price_side') not in ('bid', 'other')):
+    if conf.get("order_types", {}).get("exit") == "market" and conf.get("exit_pricing", {}).get(
+        "price_side"
+    ) not in ("bid", "other"):
         raise ConfigurationError('Market exit orders require exit_pricing.price_side = "other".')
 
 
 def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
-
-    if conf.get('stoploss') == 0.0:
+    if conf.get("stoploss") == 0.0:
         raise ConfigurationError(
-            'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
+            "The config stoploss needs to be different from 0 to avoid problems with sell orders."
         )
     # Skip if trailing stoploss is not activated
-    if not conf.get('trailing_stop', False):
+    if not conf.get("trailing_stop", False):
         return
 
-    tsl_positive = float(conf.get('trailing_stop_positive', 0))
-    tsl_offset = float(conf.get('trailing_stop_positive_offset', 0))
-    tsl_only_offset = conf.get('trailing_only_offset_is_reached', False)
+    tsl_positive = float(conf.get("trailing_stop_positive", 0))
+    tsl_offset = float(conf.get("trailing_stop_positive_offset", 0))
+    tsl_only_offset = conf.get("trailing_only_offset_is_reached", False)
 
     if tsl_only_offset:
         if tsl_positive == 0.0:
             raise ConfigurationError(
-                'The config trailing_only_offset_is_reached needs '
-                'trailing_stop_positive_offset to be more than 0 in your config.')
+                "The config trailing_only_offset_is_reached needs "
+                "trailing_stop_positive_offset to be more than 0 in your config."
+            )
     if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive:
         raise ConfigurationError(
-            'The config trailing_stop_positive_offset needs '
-            'to be greater than trailing_stop_positive in your config.')
+            "The config trailing_stop_positive_offset needs "
+            "to be greater than trailing_stop_positive in your config."
+        )
 
     # Fetch again without default
-    if 'trailing_stop_positive' in conf and float(conf['trailing_stop_positive']) == 0.0:
+    if "trailing_stop_positive" in conf and float(conf["trailing_stop_positive"]) == 0.0:
         raise ConfigurationError(
-            'The config trailing_stop_positive needs to be different from 0 '
-            'to avoid problems with sell orders.'
+            "The config trailing_stop_positive needs to be different from 0 "
+            "to avoid problems with sell orders."
         )
 
 
@@ -158,10 +165,10 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
     Edge and Dynamic whitelist should not both be enabled, since edge overrides dynamic whitelists.
     """
 
-    if not conf.get('edge', {}).get('enabled'):
+    if not conf.get("edge", {}).get("enabled"):
         return
 
-    if not conf.get('use_exit_signal', True):
+    if not conf.get("use_exit_signal", True):
         raise ConfigurationError(
             "Edge requires `use_exit_signal` to be True, otherwise no sells will happen."
         )
@@ -171,13 +178,20 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None:
     """
     Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does.
     """
-    if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT,
-                                              RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]:
+    if conf.get("runmode", RunMode.OTHER) in [
+        RunMode.OTHER,
+        RunMode.PLOT,
+        RunMode.UTIL_NO_EXCHANGE,
+        RunMode.UTIL_EXCHANGE,
+    ]:
         return
 
-    for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]):
-        if (isinstance(pl, dict) and pl.get('method') == 'StaticPairList'
-                and not conf.get('exchange', {}).get('pair_whitelist')):
+    for pl in conf.get("pairlists", [{"method": "StaticPairList"}]):
+        if (
+            isinstance(pl, dict)
+            and pl.get("method") == "StaticPairList"
+            and not conf.get("exchange", {}).get("pair_whitelist")
+        ):
             raise ConfigurationError("StaticPairList requires pair_whitelist to be set.")
 
 
@@ -186,14 +200,14 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
     Validate protection configuration validity
     """
 
-    for prot in conf.get('protections', []):
-        if ('stop_duration' in prot and 'stop_duration_candles' in prot):
+    for prot in conf.get("protections", []):
+        if "stop_duration" in prot and "stop_duration_candles" in prot:
             raise ConfigurationError(
                 "Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
             )
 
-        if ('lookback_period' in prot and 'lookback_period_candles' in prot):
+        if "lookback_period" in prot and "lookback_period_candles" in prot:
             raise ConfigurationError(
                 "Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
                 f"Please fix the protection {prot.get('method')}"
@@ -201,10 +215,10 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
 
 
 def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
-    ask_strategy = conf.get('exit_pricing', {})
-    ob_min = ask_strategy.get('order_book_min')
-    ob_max = ask_strategy.get('order_book_max')
-    if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
+    ask_strategy = conf.get("exit_pricing", {})
+    ob_min = ask_strategy.get("order_book_min")
+    ob_max = ask_strategy.get("order_book_max")
+    if ob_min is not None and ob_max is not None and ask_strategy.get("use_order_book"):
         if ob_min != ob_max:
             raise ConfigurationError(
                 "Using order_book_max != order_book_min in exit_pricing is no longer supported."
@@ -212,7 +226,7 @@ def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
             )
         else:
             # Move value to order_book_top
-            ask_strategy['order_book_top'] = ob_min
+            ask_strategy["order_book_top"] = ob_min
             logger.warning(
                 "DEPRECATED: "
                 "Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
@@ -221,7 +235,6 @@ def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
 
 
 def validate_migrated_strategy_settings(conf: Dict[str, Any]) -> None:
-
     _validate_time_in_force(conf)
     _validate_order_types(conf)
     _validate_unfilledtimeout(conf)
@@ -230,119 +243,129 @@ def validate_migrated_strategy_settings(conf: Dict[str, Any]) -> None:
 
 
 def _validate_time_in_force(conf: Dict[str, Any]) -> None:
-
-    time_in_force = conf.get('order_time_in_force', {})
-    if 'buy' in time_in_force or 'sell' in time_in_force:
-        if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
+    time_in_force = conf.get("order_time_in_force", {})
+    if "buy" in time_in_force or "sell" in time_in_force:
+        if conf.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT:
             raise ConfigurationError(
-                "Please migrate your time_in_force settings to use 'entry' and 'exit'.")
+                "Please migrate your time_in_force settings to use 'entry' and 'exit'."
+            )
         else:
             logger.warning(
                 "DEPRECATED: Using 'buy' and 'sell' for time_in_force is deprecated."
                 "Please migrate your time_in_force settings to use 'entry' and 'exit'."
             )
             process_deprecated_setting(
-                conf, 'order_time_in_force', 'buy', 'order_time_in_force', 'entry')
+                conf, "order_time_in_force", "buy", "order_time_in_force", "entry"
+            )
 
             process_deprecated_setting(
-                conf, 'order_time_in_force', 'sell', 'order_time_in_force', 'exit')
+                conf, "order_time_in_force", "sell", "order_time_in_force", "exit"
+            )
 
 
 def _validate_order_types(conf: Dict[str, Any]) -> None:
-
-    order_types = conf.get('order_types', {})
-    old_order_types = ['buy', 'sell', 'emergencysell', 'forcebuy',
-                       'forcesell', 'emergencyexit', 'forceexit', 'forceentry']
+    order_types = conf.get("order_types", {})
+    old_order_types = [
+        "buy",
+        "sell",
+        "emergencysell",
+        "forcebuy",
+        "forcesell",
+        "emergencyexit",
+        "forceexit",
+        "forceentry",
+    ]
     if any(x in order_types for x in old_order_types):
-        if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
+        if conf.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT:
             raise ConfigurationError(
-                "Please migrate your order_types settings to use the new wording.")
+                "Please migrate your order_types settings to use the new wording."
+            )
         else:
             logger.warning(
                 "DEPRECATED: Using 'buy' and 'sell' for order_types is deprecated."
                 "Please migrate your order_types settings to use 'entry' and 'exit' wording."
             )
             for o, n in [
-                ('buy', 'entry'),
-                ('sell', 'exit'),
-                ('emergencysell', 'emergency_exit'),
-                ('forcesell', 'force_exit'),
-                ('forcebuy', 'force_entry'),
-                ('emergencyexit', 'emergency_exit'),
-                ('forceexit', 'force_exit'),
-                ('forceentry', 'force_entry'),
+                ("buy", "entry"),
+                ("sell", "exit"),
+                ("emergencysell", "emergency_exit"),
+                ("forcesell", "force_exit"),
+                ("forcebuy", "force_entry"),
+                ("emergencyexit", "emergency_exit"),
+                ("forceexit", "force_exit"),
+                ("forceentry", "force_entry"),
             ]:
-
-                process_deprecated_setting(conf, 'order_types', o, 'order_types', n)
+                process_deprecated_setting(conf, "order_types", o, "order_types", n)
 
 
 def _validate_unfilledtimeout(conf: Dict[str, Any]) -> None:
-    unfilledtimeout = conf.get('unfilledtimeout', {})
-    if any(x in unfilledtimeout for x in ['buy', 'sell']):
-        if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
+    unfilledtimeout = conf.get("unfilledtimeout", {})
+    if any(x in unfilledtimeout for x in ["buy", "sell"]):
+        if conf.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT:
             raise ConfigurationError(
-                "Please migrate your unfilledtimeout settings to use the new wording.")
+                "Please migrate your unfilledtimeout settings to use the new wording."
+            )
         else:
-
             logger.warning(
                 "DEPRECATED: Using 'buy' and 'sell' for unfilledtimeout is deprecated."
                 "Please migrate your unfilledtimeout settings to use 'entry' and 'exit' wording."
             )
             for o, n in [
-                ('buy', 'entry'),
-                ('sell', 'exit'),
+                ("buy", "entry"),
+                ("sell", "exit"),
             ]:
-
-                process_deprecated_setting(conf, 'unfilledtimeout', o, 'unfilledtimeout', n)
+                process_deprecated_setting(conf, "unfilledtimeout", o, "unfilledtimeout", n)
 
 
 def _validate_pricing_rules(conf: Dict[str, Any]) -> None:
-
-    if conf.get('ask_strategy') or conf.get('bid_strategy'):
-        if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
-            raise ConfigurationError(
-                "Please migrate your pricing settings to use the new wording.")
+    if conf.get("ask_strategy") or conf.get("bid_strategy"):
+        if conf.get("trading_mode", TradingMode.SPOT) != TradingMode.SPOT:
+            raise ConfigurationError("Please migrate your pricing settings to use the new wording.")
         else:
-
             logger.warning(
                 "DEPRECATED: Using 'ask_strategy' and 'bid_strategy' is deprecated."
                 "Please migrate your settings to use 'entry_pricing' and 'exit_pricing'."
             )
-            conf['entry_pricing'] = {}
-            for obj in list(conf.get('bid_strategy', {}).keys()):
-                if obj == 'ask_last_balance':
-                    process_deprecated_setting(conf, 'bid_strategy', obj,
-                                               'entry_pricing', 'price_last_balance')
+            conf["entry_pricing"] = {}
+            for obj in list(conf.get("bid_strategy", {}).keys()):
+                if obj == "ask_last_balance":
+                    process_deprecated_setting(
+                        conf, "bid_strategy", obj, "entry_pricing", "price_last_balance"
+                    )
                 else:
-                    process_deprecated_setting(conf, 'bid_strategy', obj, 'entry_pricing', obj)
-            del conf['bid_strategy']
+                    process_deprecated_setting(conf, "bid_strategy", obj, "entry_pricing", obj)
+            del conf["bid_strategy"]
 
-            conf['exit_pricing'] = {}
-            for obj in list(conf.get('ask_strategy', {}).keys()):
-                if obj == 'bid_last_balance':
-                    process_deprecated_setting(conf, 'ask_strategy', obj,
-                                               'exit_pricing', 'price_last_balance')
+            conf["exit_pricing"] = {}
+            for obj in list(conf.get("ask_strategy", {}).keys()):
+                if obj == "bid_last_balance":
+                    process_deprecated_setting(
+                        conf, "ask_strategy", obj, "exit_pricing", "price_last_balance"
+                    )
                 else:
-                    process_deprecated_setting(conf, 'ask_strategy', obj, 'exit_pricing', obj)
-            del conf['ask_strategy']
+                    process_deprecated_setting(conf, "ask_strategy", obj, "exit_pricing", obj)
+            del conf["ask_strategy"]
 
 
 def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
-    freqai_enabled = conf.get('freqai', {}).get('enabled', False)
-    analyze_per_epoch = conf.get('analyze_per_epoch', False)
+    freqai_enabled = conf.get("freqai", {}).get("enabled", False)
+    analyze_per_epoch = conf.get("analyze_per_epoch", False)
     if analyze_per_epoch and freqai_enabled:
         raise ConfigurationError(
-            'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
+            "Using analyze-per-epoch parameter is not supported with a FreqAI strategy."
+        )
 
 
 def _validate_freqai_include_timeframes(conf: Dict[str, Any], preliminary: bool) -> None:
-    freqai_enabled = conf.get('freqai', {}).get('enabled', False)
+    freqai_enabled = conf.get("freqai", {}).get("enabled", False)
     if freqai_enabled:
-        main_tf = conf.get('timeframe', '5m')
-        freqai_include_timeframes = conf.get('freqai', {}).get('feature_parameters', {}
-                                                               ).get('include_timeframes', [])
+        main_tf = conf.get("timeframe", "5m")
+        freqai_include_timeframes = (
+            conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes", [])
+        )
 
         from freqtrade.exchange import timeframe_to_seconds
+
         main_tf_s = timeframe_to_seconds(main_tf)
         offending_lines = []
         for tf in freqai_include_timeframes:
@@ -352,57 +375,73 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any], preliminary: bool)
         if offending_lines:
             raise ConfigurationError(
                 f"Main timeframe of {main_tf} must be smaller or equal to FreqAI "
-                f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}")
+                f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}"
+            )
 
         # Ensure that the base timeframe is included in the include_timeframes list
         if not preliminary and main_tf not in freqai_include_timeframes:
-            feature_parameters = conf.get('freqai', {}).get('feature_parameters', {})
+            feature_parameters = conf.get("freqai", {}).get("feature_parameters", {})
             include_timeframes = [main_tf] + freqai_include_timeframes
-            conf.get('freqai', {}).get('feature_parameters', {}) \
-                .update({**feature_parameters, 'include_timeframes': include_timeframes})
+            conf.get("freqai", {}).get("feature_parameters", {}).update(
+                {**feature_parameters, "include_timeframes": include_timeframes}
+            )
 
 
 def _validate_freqai_backtest(conf: Dict[str, Any]) -> None:
-    if conf.get('runmode', RunMode.OTHER) == RunMode.BACKTEST:
-        freqai_enabled = conf.get('freqai', {}).get('enabled', False)
-        timerange = conf.get('timerange')
-        freqai_backtest_live_models = conf.get('freqai_backtest_live_models', False)
+    if conf.get("runmode", RunMode.OTHER) == RunMode.BACKTEST:
+        freqai_enabled = conf.get("freqai", {}).get("enabled", False)
+        timerange = conf.get("timerange")
+        freqai_backtest_live_models = conf.get("freqai_backtest_live_models", False)
         if freqai_backtest_live_models and freqai_enabled and timerange:
             raise ConfigurationError(
-                'Using timerange parameter is not supported with '
-                '--freqai-backtest-live-models parameter.')
+                "Using timerange parameter is not supported with "
+                "--freqai-backtest-live-models parameter."
+            )
 
         if freqai_backtest_live_models and not freqai_enabled:
             raise ConfigurationError(
-                'Using --freqai-backtest-live-models parameter is only '
-                'supported with a FreqAI strategy.')
+                "Using --freqai-backtest-live-models parameter is only "
+                "supported with a FreqAI strategy."
+            )
 
         if freqai_enabled and not freqai_backtest_live_models and not timerange:
             raise ConfigurationError(
-                'Please pass --timerange if you intend to use FreqAI for backtesting.')
+                "Please pass --timerange if you intend to use FreqAI for backtesting."
+            )
 
 
 def _validate_consumers(conf: Dict[str, Any]) -> None:
-    emc_conf = conf.get('external_message_consumer', {})
-    if emc_conf.get('enabled', False):
-        if len(emc_conf.get('producers', [])) < 1:
+    emc_conf = conf.get("external_message_consumer", {})
+    if emc_conf.get("enabled", False):
+        if len(emc_conf.get("producers", [])) < 1:
             raise ConfigurationError("You must specify at least 1 Producer to connect to.")
 
-        producer_names = [p['name'] for p in emc_conf.get('producers', [])]
+        producer_names = [p["name"] for p in emc_conf.get("producers", [])]
         duplicates = [item for item, count in Counter(producer_names).items() if count > 1]
         if duplicates:
             raise ConfigurationError(
-                f"Producer names must be unique. Duplicate: {', '.join(duplicates)}")
-        if conf.get('process_only_new_candles', True):
+                f"Producer names must be unique. Duplicate: {', '.join(duplicates)}"
+            )
+        if conf.get("process_only_new_candles", True):
             # Warning here or require it?
-            logger.warning("To receive best performance with external data, "
-                           "please set `process_only_new_candles` to False")
+            logger.warning(
+                "To receive best performance with external data, "
+                "please set `process_only_new_candles` to False"
+            )
+
+
+def _validate_orderflow(conf: Dict[str, Any]) -> None:
+    if conf.get("exchange", {}).get("use_public_trades"):
+        if "orderflow" not in conf:
+            raise ConfigurationError(
+                "Orderflow is a required configuration key when using public trades."
+            )
 
 
 def _strategy_settings(conf: Dict[str, Any]) -> None:
-
-    process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal')
-    process_deprecated_setting(conf, None, 'sell_profit_only', None, 'exit_profit_only')
-    process_deprecated_setting(conf, None, 'sell_profit_offset', None, 'exit_profit_offset')
-    process_deprecated_setting(conf, None, 'ignore_roi_if_buy_signal',
-                               None, 'ignore_roi_if_entry_signal')
+    process_deprecated_setting(conf, None, "use_sell_signal", None, "use_exit_signal")
+    process_deprecated_setting(conf, None, "sell_profit_only", None, "exit_profit_only")
+    process_deprecated_setting(conf, None, "sell_profit_offset", None, "exit_profit_offset")
+    process_deprecated_setting(
+        conf, None, "ignore_roi_if_buy_signal", None, "ignore_roi_if_entry_signal"
+    )

freqtrade/configuration/configuration.py

@@ -1,6 +1,8 @@
 """
 This module contains the configuration class
 """
+
+import ast
 import logging
 import warnings
 from copy import deepcopy
@@ -56,7 +58,7 @@ class Configuration:
         :return: configuration dictionary
         """
         # Keep this method as staticmethod, so it can be used from interactive environments
-        c = Configuration({'config': files}, RunMode.OTHER)
+        c = Configuration({"config": files}, RunMode.OTHER)
         return c.get_config()
 
     def load_config(self) -> Dict[str, Any]:
@@ -69,19 +71,20 @@ class Configuration:
 
         # Load environment variables
         from freqtrade.commands.arguments import NO_CONF_ALLOWED
-        if self.args.get('command') not in NO_CONF_ALLOWED:
+
+        if self.args.get("command") not in NO_CONF_ALLOWED:
             env_data = enironment_vars_to_dict()
             config = deep_merge_dicts(env_data, config)
 
         # Normalize config
-        if 'internals' not in config:
-            config['internals'] = {}
+        if "internals" not in config:
+            config["internals"] = {}
 
-        if 'pairlists' not in config:
-            config['pairlists'] = []
+        if "pairlists" not in config:
+            config["pairlists"] = []
 
         # Keep a copy of the original configuration file
-        config['original_config'] = deepcopy(config)
+        config["original_config"] = deepcopy(config)
 
         self._process_logging_options(config)
 
@@ -105,7 +108,7 @@ class Configuration:
         from freqtrade.exchange.check_exchange import check_exchange
 
         # Check if the exchange set by the user is supported
-        check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
+        check_exchange(config, config.get("experimental", {}).get("block_bad_exchanges", True))
 
         self._resolve_pairs_list(config)
 
@@ -119,52 +122,56 @@ class Configuration:
         the -v/--verbose, --logfile options
         """
         # Log level
-        config.update({'verbosity': self.args.get('verbosity', 0)})
+        config.update({"verbosity": self.args.get("verbosity", 0)})
 
-        if 'logfile' in self.args and self.args['logfile']:
-            config.update({'logfile': self.args['logfile']})
+        if "logfile" in self.args and self.args["logfile"]:
+            config.update({"logfile": self.args["logfile"]})
 
         setup_logging(config)
 
     def _process_trading_options(self, config: Config) -> None:
-        if config['runmode'] not in TRADE_MODES:
+        if config["runmode"] not in TRADE_MODES:
             return
 
-        if config.get('dry_run', False):
-            logger.info('Dry run is enabled')
-            if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]:
+        if config.get("dry_run", False):
+            logger.info("Dry run is enabled")
+            if config.get("db_url") in [None, constants.DEFAULT_DB_PROD_URL]:
                 # Default to in-memory db for dry_run if not specified
-                config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL
+                config["db_url"] = constants.DEFAULT_DB_DRYRUN_URL
         else:
-            if not config.get('db_url'):
-                config['db_url'] = constants.DEFAULT_DB_PROD_URL
-            logger.info('Dry run is disabled')
+            if not config.get("db_url"):
+                config["db_url"] = constants.DEFAULT_DB_PROD_URL
+            logger.info("Dry run is disabled")
 
         logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
 
     def _process_common_options(self, config: Config) -> None:
-
         # Set strategy if not specified in config and or if it's non default
-        if self.args.get('strategy') or not config.get('strategy'):
-            config.update({'strategy': self.args.get('strategy')})
+        if self.args.get("strategy") or not config.get("strategy"):
+            config.update({"strategy": self.args.get("strategy")})
 
-        self._args_to_config(config, argname='strategy_path',
-                             logstring='Using additional Strategy lookup path: {}')
+        self._args_to_config(
+            config, argname="strategy_path", logstring="Using additional Strategy lookup path: {}"
+        )
 
-        if ('db_url' in self.args and self.args['db_url'] and
-                self.args['db_url'] != constants.DEFAULT_DB_PROD_URL):
-            config.update({'db_url': self.args['db_url']})
-            logger.info('Parameter --db-url detected ...')
+        if (
+            "db_url" in self.args
+            and self.args["db_url"]
+            and self.args["db_url"] != constants.DEFAULT_DB_PROD_URL
+        ):
+            config.update({"db_url": self.args["db_url"]})
+            logger.info("Parameter --db-url detected ...")
 
-        self._args_to_config(config, argname='db_url_from',
-                             logstring='Parameter --db-url-from detected ...')
+        self._args_to_config(
+            config, argname="db_url_from", logstring="Parameter --db-url-from detected ..."
+        )
 
-        if config.get('force_entry_enable', False):
-            logger.warning('`force_entry_enable` RPC message enabled.')
+        if config.get("force_entry_enable", False):
+            logger.warning("`force_entry_enable` RPC message enabled.")
 
         # Support for sd_notify
-        if 'sd_notify' in self.args and self.args['sd_notify']:
-            config['internals'].update({'sd_notify': True})
+        if "sd_notify" in self.args and self.args["sd_notify"]:
+            config["internals"].update({"sd_notify": True})
 
     def _process_datadir_options(self, config: Config) -> None:
         """
@@ -172,245 +179,275 @@ class Configuration:
         --user-data, --datadir
         """
         # Check exchange parameter here - otherwise `datadir` might be wrong.
-        if 'exchange' in self.args and self.args['exchange']:
-            config['exchange']['name'] = self.args['exchange']
+        if "exchange" in self.args and self.args["exchange"]:
+            config["exchange"]["name"] = self.args["exchange"]
             logger.info(f"Using exchange {config['exchange']['name']}")
 
-        if 'pair_whitelist' not in config['exchange']:
-            config['exchange']['pair_whitelist'] = []
+        if "pair_whitelist" not in config["exchange"]:
+            config["exchange"]["pair_whitelist"] = []
 
-        if 'user_data_dir' in self.args and self.args['user_data_dir']:
-            config.update({'user_data_dir': self.args['user_data_dir']})
-        elif 'user_data_dir' not in config:
+        if "user_data_dir" in self.args and self.args["user_data_dir"]:
+            config.update({"user_data_dir": self.args["user_data_dir"]})
+        elif "user_data_dir" not in config:
             # Default to cwd/user_data (legacy option ...)
-            config.update({'user_data_dir': str(Path.cwd() / 'user_data')})
+            config.update({"user_data_dir": str(Path.cwd() / "user_data")})
 
         # reset to user_data_dir so this contains the absolute path.
-        config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False)
-        logger.info('Using user-data directory: %s ...', config['user_data_dir'])
+        config["user_data_dir"] = create_userdata_dir(config["user_data_dir"], create_dir=False)
+        logger.info("Using user-data directory: %s ...", config["user_data_dir"])
 
-        config.update({'datadir': create_datadir(config, self.args.get('datadir'))})
-        logger.info('Using data directory: %s ...', config.get('datadir'))
+        config.update({"datadir": create_datadir(config, self.args.get("datadir"))})
+        logger.info("Using data directory: %s ...", config.get("datadir"))
 
-        if self.args.get('exportfilename'):
-            self._args_to_config(config, argname='exportfilename',
-                                 logstring='Storing backtest results to {} ...')
-            config['exportfilename'] = Path(config['exportfilename'])
+        if self.args.get("exportfilename"):
+            self._args_to_config(
+                config, argname="exportfilename", logstring="Storing backtest results to {} ..."
+            )
+            config["exportfilename"] = Path(config["exportfilename"])
         else:
-            config['exportfilename'] = (config['user_data_dir']
-                                        / 'backtest_results')
+            config["exportfilename"] = config["user_data_dir"] / "backtest_results"
 
-        if self.args.get('show_sensitive'):
+        if self.args.get("show_sensitive"):
             logger.warning(
                 "Sensitive information will be shown in the upcoming output. "
                 "Please make sure to never share this output without redacting "
-                "the information yourself.")
+                "the information yourself."
+            )
 
     def _process_optimize_options(self, config: Config) -> None:
-
         # This will override the strategy configuration
-        self._args_to_config(config, argname='timeframe',
-                             logstring='Parameter -i/--timeframe detected ... '
-                                       'Using timeframe: {} ...')
-
-        self._args_to_config(config, argname='position_stacking',
-                             logstring='Parameter --enable-position-stacking detected ...')
+        self._args_to_config(
+            config,
+            argname="timeframe",
+            logstring="Parameter -i/--timeframe detected ... Using timeframe: {} ...",
+        )
 
         self._args_to_config(
-            config, argname='enable_protections',
-            logstring='Parameter --enable-protections detected, enabling Protections. ...')
+            config,
+            argname="position_stacking",
+            logstring="Parameter --enable-position-stacking detected ...",
+        )
 
-        if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]:
-            config.update({'use_max_market_positions': False})
-            logger.info('Parameter --disable-max-market-positions detected ...')
-            logger.info('max_open_trades set to unlimited ...')
-        elif 'max_open_trades' in self.args and self.args['max_open_trades']:
-            config.update({'max_open_trades': self.args['max_open_trades']})
-            logger.info('Parameter --max-open-trades detected, '
-                        'overriding max_open_trades to: %s ...', config.get('max_open_trades'))
-        elif config['runmode'] in NON_UTIL_MODES:
-            logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
+        self._args_to_config(
+            config,
+            argname="enable_protections",
+            logstring="Parameter --enable-protections detected, enabling Protections. ...",
+        )
+
+        if "use_max_market_positions" in self.args and not self.args["use_max_market_positions"]:
+            config.update({"use_max_market_positions": False})
+            logger.info("Parameter --disable-max-market-positions detected ...")
+            logger.info("max_open_trades set to unlimited ...")
+        elif "max_open_trades" in self.args and self.args["max_open_trades"]:
+            config.update({"max_open_trades": self.args["max_open_trades"]})
+            logger.info(
+                "Parameter --max-open-trades detected, overriding max_open_trades to: %s ...",
+                config.get("max_open_trades"),
+            )
+        elif config["runmode"] in NON_UTIL_MODES:
+            logger.info("Using max_open_trades: %s ...", config.get("max_open_trades"))
         # Setting max_open_trades to infinite if -1
-        if config.get('max_open_trades') == -1:
-            config['max_open_trades'] = float('inf')
+        if config.get("max_open_trades") == -1:
+            config["max_open_trades"] = float("inf")
 
-        if self.args.get('stake_amount'):
+        if self.args.get("stake_amount"):
             # Convert explicitly to float to support CLI argument for both unlimited and value
             try:
-                self.args['stake_amount'] = float(self.args['stake_amount'])
+                self.args["stake_amount"] = float(self.args["stake_amount"])
             except ValueError:
                 pass
 
         configurations = [
-            ('timeframe_detail',
-             'Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...'),
-            ('backtest_show_pair_list', 'Parameter --show-pair-list detected.'),
-            ('stake_amount',
-             'Parameter --stake-amount detected, overriding stake_amount to: {} ...'),
-            ('dry_run_wallet',
-             'Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...'),
-            ('fee', 'Parameter --fee detected, setting fee to: {} ...'),
-            ('timerange', 'Parameter --timerange detected: {} ...'),
-            ]
+            (
+                "timeframe_detail",
+                "Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...",
+            ),
+            ("backtest_show_pair_list", "Parameter --show-pair-list detected."),
+            (
+                "stake_amount",
+                "Parameter --stake-amount detected, overriding stake_amount to: {} ...",
+            ),
+            (
+                "dry_run_wallet",
+                "Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...",
+            ),
+            ("fee", "Parameter --fee detected, setting fee to: {} ..."),
+            ("timerange", "Parameter --timerange detected: {} ..."),
+        ]
 
         self._args_to_config_loop(config, configurations)
 
         self._process_datadir_options(config)
 
-        self._args_to_config(config, argname='strategy_list',
-                             logstring='Using strategy list of {} strategies', logfun=len)
+        self._args_to_config(
+            config,
+            argname="strategy_list",
+            logstring="Using strategy list of {} strategies",
+            logfun=len,
+        )
 
         configurations = [
-            ('recursive_strategy_search',
-             'Recursively searching for a strategy in the strategies folder.'),
-            ('timeframe', 'Overriding timeframe with Command line argument'),
-            ('export', 'Parameter --export detected: {} ...'),
-            ('backtest_breakdown', 'Parameter --breakdown detected ...'),
-            ('backtest_cache', 'Parameter --cache={} detected ...'),
-            ('disableparamexport', 'Parameter --disableparamexport detected: {} ...'),
-            ('freqai_backtest_live_models',
-             'Parameter --freqai-backtest-live-models detected ...'),
+            (
+                "recursive_strategy_search",
+                "Recursively searching for a strategy in the strategies folder.",
+            ),
+            ("timeframe", "Overriding timeframe with Command line argument"),
+            ("export", "Parameter --export detected: {} ..."),
+            ("backtest_breakdown", "Parameter --breakdown detected ..."),
+            ("backtest_cache", "Parameter --cache={} detected ..."),
+            ("disableparamexport", "Parameter --disableparamexport detected: {} ..."),
+            ("freqai_backtest_live_models", "Parameter --freqai-backtest-live-models detected ..."),
         ]
         self._args_to_config_loop(config, configurations)
 
         # Edge section:
-        if 'stoploss_range' in self.args and self.args["stoploss_range"]:
-            txt_range = eval(self.args["stoploss_range"])
-            config['edge'].update({'stoploss_range_min': txt_range[0]})
-            config['edge'].update({'stoploss_range_max': txt_range[1]})
-            config['edge'].update({'stoploss_range_step': txt_range[2]})
-            logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"])
+        if "stoploss_range" in self.args and self.args["stoploss_range"]:
+            txt_range = ast.literal_eval(self.args["stoploss_range"])
+            config["edge"].update({"stoploss_range_min": txt_range[0]})
+            config["edge"].update({"stoploss_range_max": txt_range[1]})
+            config["edge"].update({"stoploss_range_step": txt_range[2]})
+            logger.info("Parameter --stoplosses detected: %s ...", self.args["stoploss_range"])
 
         # Hyperopt section
 
         configurations = [
-            ('hyperopt', 'Using Hyperopt class name: {}'),
-            ('hyperopt_path', 'Using additional Hyperopt lookup path: {}'),
-            ('hyperoptexportfilename', 'Using hyperopt file: {}'),
-            ('lookahead_analysis_exportfilename', 'Saving lookahead analysis results into {} ...'),
-            ('epochs', 'Parameter --epochs detected ... Will run Hyperopt with for {} epochs ...'),
-            ('spaces', 'Parameter -s/--spaces detected: {}'),
-            ('analyze_per_epoch', 'Parameter --analyze-per-epoch detected.'),
-            ('print_all', 'Parameter --print-all detected ...'),
+            ("hyperopt", "Using Hyperopt class name: {}"),
+            ("hyperopt_path", "Using additional Hyperopt lookup path: {}"),
+            ("hyperoptexportfilename", "Using hyperopt file: {}"),
+            ("lookahead_analysis_exportfilename", "Saving lookahead analysis results into {} ..."),
+            ("epochs", "Parameter --epochs detected ... Will run Hyperopt with for {} epochs ..."),
+            ("spaces", "Parameter -s/--spaces detected: {}"),
+            ("analyze_per_epoch", "Parameter --analyze-per-epoch detected."),
+            ("print_all", "Parameter --print-all detected ..."),
         ]
         self._args_to_config_loop(config, configurations)
 
-        if 'print_colorized' in self.args and not self.args["print_colorized"]:
-            logger.info('Parameter --no-color detected ...')
-            config.update({'print_colorized': False})
+        if "print_colorized" in self.args and not self.args["print_colorized"]:
+            logger.info("Parameter --no-color detected ...")
+            config.update({"print_colorized": False})
         else:
-            config.update({'print_colorized': True})
+            config.update({"print_colorized": True})
 
         configurations = [
-            ('print_json', 'Parameter --print-json detected ...'),
-            ('export_csv', 'Parameter --export-csv detected: {}'),
-            ('hyperopt_jobs', 'Parameter -j/--job-workers detected: {}'),
-            ('hyperopt_random_state', 'Parameter --random-state detected: {}'),
-            ('hyperopt_min_trades', 'Parameter --min-trades detected: {}'),
-            ('hyperopt_loss', 'Using Hyperopt loss class name: {}'),
-            ('hyperopt_show_index', 'Parameter -n/--index detected: {}'),
-            ('hyperopt_list_best', 'Parameter --best detected: {}'),
-            ('hyperopt_list_profitable', 'Parameter --profitable detected: {}'),
-            ('hyperopt_list_min_trades', 'Parameter --min-trades detected: {}'),
-            ('hyperopt_list_max_trades', 'Parameter --max-trades detected: {}'),
-            ('hyperopt_list_min_avg_time', 'Parameter --min-avg-time detected: {}'),
-            ('hyperopt_list_max_avg_time', 'Parameter --max-avg-time detected: {}'),
-            ('hyperopt_list_min_avg_profit', 'Parameter --min-avg-profit detected: {}'),
-            ('hyperopt_list_max_avg_profit', 'Parameter --max-avg-profit detected: {}'),
-            ('hyperopt_list_min_total_profit', 'Parameter --min-total-profit detected: {}'),
-            ('hyperopt_list_max_total_profit', 'Parameter --max-total-profit detected: {}'),
-            ('hyperopt_list_min_objective', 'Parameter --min-objective detected: {}'),
-            ('hyperopt_list_max_objective', 'Parameter --max-objective detected: {}'),
-            ('hyperopt_list_no_details', 'Parameter --no-details detected: {}'),
-            ('hyperopt_show_no_header', 'Parameter --no-header detected: {}'),
-            ('hyperopt_ignore_missing_space', 'Paramter --ignore-missing-space detected: {}'),
+            ("print_json", "Parameter --print-json detected ..."),
+            ("export_csv", "Parameter --export-csv detected: {}"),
+            ("hyperopt_jobs", "Parameter -j/--job-workers detected: {}"),
+            ("hyperopt_random_state", "Parameter --random-state detected: {}"),
+            ("hyperopt_min_trades", "Parameter --min-trades detected: {}"),
+            ("hyperopt_loss", "Using Hyperopt loss class name: {}"),
+            ("hyperopt_show_index", "Parameter -n/--index detected: {}"),
+            ("hyperopt_list_best", "Parameter --best detected: {}"),
+            ("hyperopt_list_profitable", "Parameter --profitable detected: {}"),
+            ("hyperopt_list_min_trades", "Parameter --min-trades detected: {}"),
+            ("hyperopt_list_max_trades", "Parameter --max-trades detected: {}"),
+            ("hyperopt_list_min_avg_time", "Parameter --min-avg-time detected: {}"),
+            ("hyperopt_list_max_avg_time", "Parameter --max-avg-time detected: {}"),
+            ("hyperopt_list_min_avg_profit", "Parameter --min-avg-profit detected: {}"),
+            ("hyperopt_list_max_avg_profit", "Parameter --max-avg-profit detected: {}"),
+            ("hyperopt_list_min_total_profit", "Parameter --min-total-profit detected: {}"),
+            ("hyperopt_list_max_total_profit", "Parameter --max-total-profit detected: {}"),
+            ("hyperopt_list_min_objective", "Parameter --min-objective detected: {}"),
+            ("hyperopt_list_max_objective", "Parameter --max-objective detected: {}"),
+            ("hyperopt_list_no_details", "Parameter --no-details detected: {}"),
+            ("hyperopt_show_no_header", "Parameter --no-header detected: {}"),
+            ("hyperopt_ignore_missing_space", "Parameter --ignore-missing-space detected: {}"),
         ]
 
         self._args_to_config_loop(config, configurations)
 
     def _process_plot_options(self, config: Config) -> None:
-
         configurations = [
-            ('pairs', 'Using pairs {}'),
-            ('indicators1', 'Using indicators1: {}'),
-            ('indicators2', 'Using indicators2: {}'),
-            ('trade_ids', 'Filtering on trade_ids: {}'),
-            ('plot_limit', 'Limiting plot to: {}'),
-            ('plot_auto_open', 'Parameter --auto-open detected.'),
-            ('trade_source', 'Using trades from: {}'),
-            ('prepend_data', 'Prepend detected. Allowing data prepending.'),
-            ('erase', 'Erase detected. Deleting existing data.'),
-            ('no_trades', 'Parameter --no-trades detected.'),
-            ('timeframes', 'timeframes --timeframes: {}'),
-            ('days', 'Detected --days: {}'),
-            ('include_inactive', 'Detected --include-inactive-pairs: {}'),
-            ('download_trades', 'Detected --dl-trades: {}'),
-            ('dataformat_ohlcv', 'Using "{}" to store OHLCV data.'),
-            ('dataformat_trades', 'Using "{}" to store trades data.'),
-            ('show_timerange', 'Detected --show-timerange'),
+            ("pairs", "Using pairs {}"),
+            ("indicators1", "Using indicators1: {}"),
+            ("indicators2", "Using indicators2: {}"),
+            ("trade_ids", "Filtering on trade_ids: {}"),
+            ("plot_limit", "Limiting plot to: {}"),
+            ("plot_auto_open", "Parameter --auto-open detected."),
+            ("trade_source", "Using trades from: {}"),
+            ("prepend_data", "Prepend detected. Allowing data prepending."),
+            ("erase", "Erase detected. Deleting existing data."),
+            ("no_trades", "Parameter --no-trades detected."),
+            ("timeframes", "timeframes --timeframes: {}"),
+            ("days", "Detected --days: {}"),
+            ("include_inactive", "Detected --include-inactive-pairs: {}"),
+            ("download_trades", "Detected --dl-trades: {}"),
+            ("convert_trades", "Detected --convert: {} - Converting Trade data to OHCV {}"),
+            ("dataformat_ohlcv", 'Using "{}" to store OHLCV data.'),
+            ("dataformat_trades", 'Using "{}" to store trades data.'),
+            ("show_timerange", "Detected --show-timerange"),
         ]
         self._args_to_config_loop(config, configurations)
 
     def _process_data_options(self, config: Config) -> None:
-        self._args_to_config(config, argname='new_pairs_days',
-                             logstring='Detected --new-pairs-days: {}')
-        self._args_to_config(config, argname='trading_mode',
-                             logstring='Detected --trading-mode: {}')
-        config['candle_type_def'] = CandleType.get_default(
-            config.get('trading_mode', 'spot') or 'spot')
-        config['trading_mode'] = TradingMode(config.get('trading_mode', 'spot') or 'spot')
-        self._args_to_config(config, argname='candle_types',
-                             logstring='Detected --candle-types: {}')
+        self._args_to_config(
+            config, argname="new_pairs_days", logstring="Detected --new-pairs-days: {}"
+        )
+        self._args_to_config(
+            config, argname="trading_mode", logstring="Detected --trading-mode: {}"
+        )
+        config["candle_type_def"] = CandleType.get_default(
+            config.get("trading_mode", "spot") or "spot"
+        )
+        config["trading_mode"] = TradingMode(config.get("trading_mode", "spot") or "spot")
+        self._args_to_config(
+            config, argname="candle_types", logstring="Detected --candle-types: {}"
+        )
 
     def _process_analyze_options(self, config: Config) -> None:
         configurations = [
-            ('analysis_groups', 'Analysis reason groups: {}'),
-            ('enter_reason_list', 'Analysis enter tag list: {}'),
-            ('exit_reason_list', 'Analysis exit tag list: {}'),
-            ('indicator_list', 'Analysis indicator list: {}'),
-            ('timerange', 'Filter trades by timerange: {}'),
-            ('analysis_rejected', 'Analyse rejected signals: {}'),
-            ('analysis_to_csv', 'Store analysis tables to CSV: {}'),
-            ('analysis_csv_path', 'Path to store analysis CSVs: {}'),
+            ("analysis_groups", "Analysis reason groups: {}"),
+            ("enter_reason_list", "Analysis enter tag list: {}"),
+            ("exit_reason_list", "Analysis exit tag list: {}"),
+            ("indicator_list", "Analysis indicator list: {}"),
+            ("timerange", "Filter trades by timerange: {}"),
+            ("analysis_rejected", "Analyse rejected signals: {}"),
+            ("analysis_to_csv", "Store analysis tables to CSV: {}"),
+            ("analysis_csv_path", "Path to store analysis CSVs: {}"),
             # Lookahead analysis results
-            ('targeted_trade_amount', 'Targeted Trade amount: {}'),
-            ('minimum_trade_amount', 'Minimum Trade amount: {}'),
-            ('lookahead_analysis_exportfilename', 'Path to store lookahead-analysis-results: {}'),
-            ('startup_candle', 'Startup candle to be used on recursive analysis: {}'),
+            ("targeted_trade_amount", "Targeted Trade amount: {}"),
+            ("minimum_trade_amount", "Minimum Trade amount: {}"),
+            ("lookahead_analysis_exportfilename", "Path to store lookahead-analysis-results: {}"),
+            ("startup_candle", "Startup candle to be used on recursive analysis: {}"),
         ]
         self._args_to_config_loop(config, configurations)
 
     def _args_to_config_loop(self, config, configurations: List[Tuple[str, str]]) -> None:
-
         for argname, logstring in configurations:
             self._args_to_config(config, argname=argname, logstring=logstring)
 
     def _process_runmode(self, config: Config) -> None:
-
-        self._args_to_config(config, argname='dry_run',
-                             logstring='Parameter --dry-run detected, '
-                             'overriding dry_run to: {} ...')
+        self._args_to_config(
+            config,
+            argname="dry_run",
+            logstring="Parameter --dry-run detected, overriding dry_run to: {} ...",
+        )
 
         if not self.runmode:
             # Handle real mode, infer dry/live from config
-            self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE
+            self.runmode = RunMode.DRY_RUN if config.get("dry_run", True) else RunMode.LIVE
             logger.info(f"Runmode set to {self.runmode.value}.")
 
-        config.update({'runmode': self.runmode})
+        config.update({"runmode": self.runmode})
 
     def _process_freqai_options(self, config: Config) -> None:
+        self._args_to_config(
+            config, argname="freqaimodel", logstring="Using freqaimodel class name: {}"
+        )
 
-        self._args_to_config(config, argname='freqaimodel',
-                             logstring='Using freqaimodel class name: {}')
-
-        self._args_to_config(config, argname='freqaimodel_path',
-                             logstring='Using freqaimodel path: {}')
+        self._args_to_config(
+            config, argname="freqaimodel_path", logstring="Using freqaimodel path: {}"
+        )
 
         return
 
-    def _args_to_config(self, config: Config, argname: str,
-                        logstring: str, logfun: Optional[Callable] = None,
-                        deprecated_msg: Optional[str] = None) -> None:
+    def _args_to_config(
+        self,
+        config: Config,
+        argname: str,
+        logstring: str,
+        logfun: Optional[Callable] = None,
+        deprecated_msg: Optional[str] = None,
+    ) -> None:
         """
         :param config: Configuration dictionary
         :param argname: Argumentname in self.args - will be copied to config dict.
@@ -420,9 +457,11 @@ class Configuration:
                         sample: logfun=len (prints the length of the found
                         configuration instead of the content)
         """
-        if (argname in self.args and self.args[argname] is not None
-                and self.args[argname] is not False):
-
+        if (
+            argname in self.args
+            and self.args[argname] is not None
+            and self.args[argname] is not False
+        ):
             config.update({argname: self.args[argname]})
             if logfun:
                 logger.info(logstring.format(logfun(config[argname])))
@@ -441,7 +480,7 @@ class Configuration:
         """
 
         if "pairs" in config:
-            config['exchange']['pair_whitelist'] = config['pairs']
+            config["exchange"]["pair_whitelist"] = config["pairs"]
             return
 
         if "pairs_file" in self.args and self.args["pairs_file"]:
@@ -451,19 +490,19 @@ class Configuration:
             # or if pairs file is specified explicitly
             if not pairs_file.exists():
                 raise OperationalException(f'No pairs file found with path "{pairs_file}".')
-            config['pairs'] = load_file(pairs_file)
-            if isinstance(config['pairs'], list):
-                config['pairs'].sort()
+            config["pairs"] = load_file(pairs_file)
+            if isinstance(config["pairs"], list):
+                config["pairs"].sort()
             return
 
-        if 'config' in self.args and self.args['config']:
+        if "config" in self.args and self.args["config"]:
             logger.info("Using pairlist from configuration.")
-            config['pairs'] = config.get('exchange', {}).get('pair_whitelist')
+            config["pairs"] = config.get("exchange", {}).get("pair_whitelist")
         else:
             # Fall back to /dl_path/pairs.json
-            pairs_file = config['datadir'] / 'pairs.json'
+            pairs_file = config["datadir"] / "pairs.json"
             if pairs_file.exists():
                 logger.info(f'Reading pairs file "{pairs_file}".')
-                config['pairs'] = load_file(pairs_file)
-                if 'pairs' in config and isinstance(config['pairs'], list):
-                    config['pairs'].sort()
+                config["pairs"] = load_file(pairs_file)
+                if "pairs" in config and isinstance(config["pairs"], list):
+                    config["pairs"].sort()

freqtrade/configuration/deprecated_settings.py

@@ -12,9 +12,13 @@ from freqtrade.exceptions import ConfigurationError, OperationalException
 logger = logging.getLogger(__name__)
 
 
-def check_conflicting_settings(config: Config,
-                               section_old: Optional[str], name_old: str,
-                               section_new: Optional[str], name_new: str) -> None:
+def check_conflicting_settings(
+    config: Config,
+    section_old: Optional[str],
+    name_old: str,
+    section_new: Optional[str],
+    name_new: str,
+) -> None:
     section_new_config = config.get(section_new, {}) if section_new else config
     section_old_config = config.get(section_old, {}) if section_old else config
     if name_new in section_new_config and name_old in section_old_config:
@@ -29,9 +33,9 @@ def check_conflicting_settings(config: Config,
         )
 
 
-def process_removed_setting(config: Config,
-                            section1: str, name1: str,
-                            section2: Optional[str], name2: str) -> None:
+def process_removed_setting(
+    config: Config, section1: str, name1: str, section2: Optional[str], name2: str
+) -> None:
     """
     :param section1: Removed section
     :param name1: Removed setting name
@@ -48,10 +52,13 @@ def process_removed_setting(config: Config,
         )
 
 
-def process_deprecated_setting(config: Config,
-                               section_old: Optional[str], name_old: str,
-                               section_new: Optional[str], name_new: str
-                               ) -> None:
+def process_deprecated_setting(
+    config: Config,
+    section_old: Optional[str],
+    name_old: str,
+    section_new: Optional[str],
+    name_new: str,
+) -> None:
     check_conflicting_settings(config, section_old, name_old, section_new, name_new)
     section_old_config = config.get(section_old, {}) if section_old else config
 
@@ -71,57 +78,91 @@ def process_deprecated_setting(config: Config,
 
 
 def process_temporary_deprecated_settings(config: Config) -> None:
-
     # Kept for future deprecated / moved settings
     # check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
     #                            'experimental', 'use_sell_signal')
 
-    process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
-                               None, 'ignore_buying_expired_candle_after')
+    process_deprecated_setting(
+        config,
+        "ask_strategy",
+        "ignore_buying_expired_candle_after",
+        None,
+        "ignore_buying_expired_candle_after",
+    )
 
-    process_deprecated_setting(config, None, 'forcebuy_enable', None, 'force_entry_enable')
+    process_deprecated_setting(config, None, "forcebuy_enable", None, "force_entry_enable")
 
     # New settings
-    if config.get('telegram'):
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'sell',
-                                   'notification_settings', 'exit')
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'sell_fill',
-                                   'notification_settings', 'exit_fill')
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'sell_cancel',
-                                   'notification_settings', 'exit_cancel')
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'buy',
-                                   'notification_settings', 'entry')
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'buy_fill',
-                                   'notification_settings', 'entry_fill')
-        process_deprecated_setting(config['telegram'], 'notification_settings', 'buy_cancel',
-                                   'notification_settings', 'entry_cancel')
-    if config.get('webhook'):
-        process_deprecated_setting(config, 'webhook', 'webhookbuy', 'webhook', 'webhookentry')
-        process_deprecated_setting(config, 'webhook', 'webhookbuycancel',
-                                   'webhook', 'webhookentrycancel')
-        process_deprecated_setting(config, 'webhook', 'webhookbuyfill',
-                                   'webhook', 'webhookentryfill')
-        process_deprecated_setting(config, 'webhook', 'webhooksell', 'webhook', 'webhookexit')
-        process_deprecated_setting(config, 'webhook', 'webhooksellcancel',
-                                   'webhook', 'webhookexitcancel')
-        process_deprecated_setting(config, 'webhook', 'webhooksellfill',
-                                   'webhook', 'webhookexitfill')
+    if config.get("telegram"):
+        process_deprecated_setting(
+            config["telegram"], "notification_settings", "sell", "notification_settings", "exit"
+        )
+        process_deprecated_setting(
+            config["telegram"],
+            "notification_settings",
+            "sell_fill",
+            "notification_settings",
+            "exit_fill",
+        )
+        process_deprecated_setting(
+            config["telegram"],
+            "notification_settings",
+            "sell_cancel",
+            "notification_settings",
+            "exit_cancel",
+        )
+        process_deprecated_setting(
+            config["telegram"], "notification_settings", "buy", "notification_settings", "entry"
+        )
+        process_deprecated_setting(
+            config["telegram"],
+            "notification_settings",
+            "buy_fill",
+            "notification_settings",
+            "entry_fill",
+        )
+        process_deprecated_setting(
+            config["telegram"],
+            "notification_settings",
+            "buy_cancel",
+            "notification_settings",
+            "entry_cancel",
+        )
+    if config.get("webhook"):
+        process_deprecated_setting(config, "webhook", "webhookbuy", "webhook", "webhookentry")
+        process_deprecated_setting(
+            config, "webhook", "webhookbuycancel", "webhook", "webhookentrycancel"
+        )
+        process_deprecated_setting(
+            config, "webhook", "webhookbuyfill", "webhook", "webhookentryfill"
+        )
+        process_deprecated_setting(config, "webhook", "webhooksell", "webhook", "webhookexit")
+        process_deprecated_setting(
+            config, "webhook", "webhooksellcancel", "webhook", "webhookexitcancel"
+        )
+        process_deprecated_setting(
+            config, "webhook", "webhooksellfill", "webhook", "webhookexitfill"
+        )
 
     # Legacy way - having them in experimental ...
 
-    process_removed_setting(config, 'experimental', 'use_sell_signal', None, 'use_exit_signal')
-    process_removed_setting(config, 'experimental', 'sell_profit_only', None, 'exit_profit_only')
-    process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
-                            None, 'ignore_roi_if_entry_signal')
+    process_removed_setting(config, "experimental", "use_sell_signal", None, "use_exit_signal")
+    process_removed_setting(config, "experimental", "sell_profit_only", None, "exit_profit_only")
+    process_removed_setting(
+        config, "experimental", "ignore_roi_if_buy_signal", None, "ignore_roi_if_entry_signal"
+    )
 
-    process_removed_setting(config, 'ask_strategy', 'use_sell_signal', None, 'use_exit_signal')
-    process_removed_setting(config, 'ask_strategy', 'sell_profit_only', None, 'exit_profit_only')
-    process_removed_setting(config, 'ask_strategy', 'sell_profit_offset',
-                            None, 'exit_profit_offset')
-    process_removed_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
-                            None, 'ignore_roi_if_entry_signal')
-    if (config.get('edge', {}).get('enabled', False)
-       and 'capital_available_percentage' in config.get('edge', {})):
+    process_removed_setting(config, "ask_strategy", "use_sell_signal", None, "use_exit_signal")
+    process_removed_setting(config, "ask_strategy", "sell_profit_only", None, "exit_profit_only")
+    process_removed_setting(
+        config, "ask_strategy", "sell_profit_offset", None, "exit_profit_offset"
+    )
+    process_removed_setting(
+        config, "ask_strategy", "ignore_roi_if_buy_signal", None, "ignore_roi_if_entry_signal"
+    )
+    if config.get("edge", {}).get(
+        "enabled", False
+    ) and "capital_available_percentage" in config.get("edge", {}):
         raise ConfigurationError(
             "DEPRECATED: "
             "Using 'edge.capital_available_percentage' has been deprecated in favor of "
@@ -129,12 +170,11 @@ def process_temporary_deprecated_settings(config: Config) -> None:
             "'tradable_balance_ratio' and remove 'capital_available_percentage' "
             "from the edge configuration."
         )
-    if 'ticker_interval' in config:
-
+    if "ticker_interval" in config:
         raise ConfigurationError(
             "DEPRECATED: 'ticker_interval' detected. "
             "Please use 'timeframe' instead of 'ticker_interval."
         )
 
-    if 'protections' in config:
+    if "protections" in config:
         logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

freqtrade/configuration/detect_environment.py

@@ -5,4 +5,4 @@ def running_in_docker() -> bool:
     """
     Check if we are running in a docker container
     """
-    return os.environ.get('FT_APP_ENV') == 'docker'
+    return os.environ.get("FT_APP_ENV") == "docker"

freqtrade/configuration/directory_operations.py

@@ -4,8 +4,14 @@ from pathlib import Path
 from typing import Optional
 
 from freqtrade.configuration.detect_environment import running_in_docker
-from freqtrade.constants import (USER_DATA_FILES, USERPATH_FREQAIMODELS, USERPATH_HYPEROPTS,
-                                 USERPATH_NOTEBOOKS, USERPATH_STRATEGIES, Config)
+from freqtrade.constants import (
+    USER_DATA_FILES,
+    USERPATH_FREQAIMODELS,
+    USERPATH_HYPEROPTS,
+    USERPATH_NOTEBOOKS,
+    USERPATH_STRATEGIES,
+    Config,
+)
 from freqtrade.exceptions import OperationalException
 
 
@@ -13,16 +19,15 @@ logger = logging.getLogger(__name__)
 
 
 def create_datadir(config: Config, datadir: Optional[str] = None) -> Path:
-
     folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data")
     if not datadir:
         # set datadir
-        exchange_name = config.get('exchange', {}).get('name', '').lower()
+        exchange_name = config.get("exchange", {}).get("name", "").lower()
         folder = folder.joinpath(exchange_name)
 
     if not folder.is_dir():
         folder.mkdir(parents=True)
-        logger.info(f'Created data directory: {datadir}')
+        logger.info(f"Created data directory: {datadir}")
     return folder
 
 
@@ -33,9 +38,9 @@ def chown_user_directory(directory: Path) -> None:
     """
     if running_in_docker():
         try:
-            import subprocess
-            subprocess.check_output(
-                ['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
+            import subprocess  # noqa: S404
+
+            subprocess.check_output(["sudo", "chown", "-R", "ftuser:", str(directory.resolve())])
         except Exception:
             logger.warning(f"Could not chown {directory}")
 
@@ -50,18 +55,28 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     :param create_dir: Create directory if it does not exist.
     :return: Path object containing the directory
     """
-    sub_dirs = ["backtest_results", "data", USERPATH_HYPEROPTS, "hyperopt_results", "logs",
-                USERPATH_NOTEBOOKS, "plot", USERPATH_STRATEGIES, USERPATH_FREQAIMODELS]
+    sub_dirs = [
+        "backtest_results",
+        "data",
+        USERPATH_HYPEROPTS,
+        "hyperopt_results",
+        "logs",
+        USERPATH_NOTEBOOKS,
+        "plot",
+        USERPATH_STRATEGIES,
+        USERPATH_FREQAIMODELS,
+    ]
     folder = Path(directory)
     chown_user_directory(folder)
     if not folder.is_dir():
         if create_dir:
             folder.mkdir(parents=True)
-            logger.info(f'Created user-data directory: {folder}')
+            logger.info(f"Created user-data directory: {folder}")
         else:
             raise OperationalException(
                 f"Directory `{folder}` does not exist. "
-                "Please use `freqtrade create-userdir` to create a user directory")
+                "Please use `freqtrade create-userdir` to create a user directory"
+            )
 
     # Create required subdirectories
     for f in sub_dirs:

freqtrade/configuration/environment_vars.py

@@ -16,9 +16,9 @@ def _get_var_typed(val):
         try:
             return float(val)
         except ValueError:
-            if val.lower() in ('t', 'true'):
+            if val.lower() in ("t", "true"):
                 return True
-            elif val.lower() in ('f', 'false'):
+            elif val.lower() in ("f", "false"):
                 return False
     # keep as string
     return val
@@ -32,16 +32,21 @@ def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str
     :param prefix: Prefix to consider (usually FREQTRADE__)
     :return: Nested dict based on available and relevant variables.
     """
-    no_convert = ['CHAT_ID', 'PASSWORD']
+    no_convert = ["CHAT_ID", "PASSWORD"]
     relevant_vars: Dict[str, Any] = {}
 
     for env_var, val in sorted(env_dict.items()):
         if env_var.startswith(prefix):
             logger.info(f"Loading variable '{env_var}'")
-            key = env_var.replace(prefix, '')
-            for k in reversed(key.split('__')):
-                val = {k.lower(): _get_var_typed(val)
-                       if not isinstance(val, dict) and k not in no_convert else val}
+            key = env_var.replace(prefix, "")
+            for k in reversed(key.split("__")):
+                val = {
+                    k.lower(): (
+                        _get_var_typed(val)
+                        if not isinstance(val, dict) and k not in no_convert
+                        else val
+                    )
+                }
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
 

freqtrade/configuration/load_config.py

@@ -1,6 +1,7 @@
 """
 This module contain functions to load the configuration file
 """
+
 import logging
 import re
 import sys
@@ -25,25 +26,25 @@ def log_config_error_range(path: str, errmsg: str) -> str:
     """
     Parses configuration file and prints range around error
     """
-    if path != '-':
-        offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
+    if path != "-":
+        offsetlist = re.findall(r"(?<=Parse\serror\sat\soffset\s)\d+", errmsg)
         if offsetlist:
             offset = int(offsetlist[0])
             text = Path(path).read_text()
             # Fetch an offset of 80 characters around the error line
-            subtext = text[offset - min(80, offset):offset + 80]
-            segments = subtext.split('\n')
+            subtext = text[offset - min(80, offset) : offset + 80]
+            segments = subtext.split("\n")
             if len(segments) > 3:
                 # Remove first and last lines, to avoid odd truncations
-                return '\n'.join(segments[1:-1])
+                return "\n".join(segments[1:-1])
             else:
                 return subtext
-    return ''
+    return ""
 
 
 def load_file(path: Path) -> Dict[str, Any]:
     try:
-        with path.open('r') as file:
+        with path.open("r") as file:
             config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
         raise OperationalException(f'File "{path}" not found!') from None
@@ -58,25 +59,27 @@ def load_config_file(path: str) -> Dict[str, Any]:
     """
     try:
         # Read config from stdin if requested in the options
-        with Path(path).open() if path != '-' else sys.stdin as file:
+        with Path(path).open() if path != "-" else sys.stdin as file:
             config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
         raise OperationalException(
             f'Config file "{path}" not found!'
-            ' Please create a config file or check whether it exists.') from None
+            " Please create a config file or check whether it exists."
+        ) from None
     except rapidjson.JSONDecodeError as e:
         err_range = log_config_error_range(path, str(e))
         raise ConfigurationError(
-            f'{e}\n'
-            f'Please verify the following segment of your configuration:\n{err_range}'
-            if err_range else 'Please verify your configuration file for syntax errors.'
+            f"{e}\nPlease verify the following segment of your configuration:\n{err_range}"
+            if err_range
+            else "Please verify your configuration file for syntax errors."
         )
 
     return config
 
 
 def load_from_files(
-        files: List[str], base_path: Optional[Path] = None, level: int = 0) -> Dict[str, Any]:
+    files: List[str], base_path: Optional[Path] = None, level: int = 0
+) -> Dict[str, Any]:
     """
     Recursively load configuration files if specified.
     Sub-files are assumed to be relative to the initial config.
@@ -90,8 +93,8 @@ def load_from_files(
     files_loaded = []
     # We expect here a list of config filenames
     for filename in files:
-        logger.info(f'Using config: {filename} ...')
-        if filename == '-':
+        logger.info(f"Using config: {filename} ...")
+        if filename == "-":
             # Immediately load stdin and return
             return load_config_file(filename)
         file = Path(filename)
@@ -100,10 +103,11 @@ def load_from_files(
             file = base_path / file
 
         config_tmp = load_config_file(str(file))
-        if 'add_config_files' in config_tmp:
+        if "add_config_files" in config_tmp:
             config_sub = load_from_files(
-                config_tmp['add_config_files'], file.resolve().parent, level + 1)
-            files_loaded.extend(config_sub.get('config_files', []))
+                config_tmp["add_config_files"], file.resolve().parent, level + 1
+            )
+            files_loaded.extend(config_sub.get("config_files", []))
             config_tmp = deep_merge_dicts(config_tmp, config_sub)
 
         files_loaded.insert(0, str(file))
@@ -111,6 +115,6 @@ def load_from_files(
         # Merge config options, overwriting prior values
         config = deep_merge_dicts(config_tmp, config)
 
-    config['config_files'] = files_loaded
+    config["config_files"] = files_loaded
 
     return config

freqtrade/configuration/timerange.py

@@ -1,6 +1,7 @@
 """
 This module contains the argument manager class
 """
+
 import logging
 import re
 from datetime import datetime, timezone
@@ -22,9 +23,13 @@ class TimeRange:
     if *type is None, don't use corresponding startvalue.
     """
 
-    def __init__(self, starttype: Optional[str] = None, stoptype: Optional[str] = None,
-                 startts: int = 0, stopts: int = 0):
-
+    def __init__(
+        self,
+        starttype: Optional[str] = None,
+        stoptype: Optional[str] = None,
+        startts: int = 0,
+        stopts: int = 0,
+    ):
         self.starttype: Optional[str] = starttype
         self.stoptype: Optional[str] = stoptype
         self.startts: int = startts
@@ -48,12 +53,12 @@ class TimeRange:
         Returns a string representation of the timerange as used by parse_timerange.
         Follows the format yyyymmdd-yyyymmdd - leaving out the parts that are not set.
         """
-        start = ''
-        stop = ''
+        start = ""
+        stop = ""
         if startdt := self.startdt:
-            start = startdt.strftime('%Y%m%d')
+            start = startdt.strftime("%Y%m%d")
         if stopdt := self.stopdt:
-            stop = stopdt.strftime('%Y%m%d')
+            stop = stopdt.strftime("%Y%m%d")
         return f"{start}-{stop}"
 
     @property
@@ -61,7 +66,7 @@ class TimeRange:
         """
         Returns a string representation of the start date
         """
-        val = 'unbounded'
+        val = "unbounded"
         if (startdt := self.startdt) is not None:
             val = startdt.strftime(DATETIME_PRINT_FORMAT)
         return val
@@ -71,15 +76,19 @@ class TimeRange:
         """
         Returns a string representation of the stop date
         """
-        val = 'unbounded'
+        val = "unbounded"
         if (stopdt := self.stopdt) is not None:
             val = stopdt.strftime(DATETIME_PRINT_FORMAT)
         return val
 
     def __eq__(self, other):
         """Override the default Equals behavior"""
-        return (self.starttype == other.starttype and self.stoptype == other.stoptype
-                and self.startts == other.startts and self.stopts == other.stopts)
+        return (
+            self.starttype == other.starttype
+            and self.stoptype == other.stoptype
+            and self.startts == other.startts
+            and self.stopts == other.stopts
+        )
 
     def subtract_start(self, seconds: int) -> None:
         """
@@ -90,8 +99,9 @@ class TimeRange:
         if self.startts:
             self.startts = self.startts - seconds
 
-    def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
-                                  min_date: datetime) -> None:
+    def adjust_start_if_necessary(
+        self, timeframe_secs: int, startup_candles: int, min_date: datetime
+    ) -> None:
         """
         Adjust startts by <startup_candles> candles.
         Applies only if no startup-candles have been available.
@@ -101,13 +111,13 @@ class TimeRange:
                          has to be moved
         :return: None (Modifies the object in place)
         """
-        if (not self.starttype or (startup_candles
-                                   and min_date.timestamp() >= self.startts)):
+        if not self.starttype or (startup_candles and min_date.timestamp() >= self.startts):
             # If no startts was defined, or backtest-data starts at the defined backtest-date
-            logger.warning("Moving start-date by %s candles to account for startup time.",
-                           startup_candles)
+            logger.warning(
+                "Moving start-date by %s candles to account for startup time.", startup_candles
+            )
             self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
-            self.starttype = 'date'
+            self.starttype = "date"
 
     @classmethod
     def parse_timerange(cls, text: Optional[str]) -> Self:
@@ -118,16 +128,17 @@ class TimeRange:
         """
         if not text:
             return cls(None, None, 0, 0)
-        syntax = [(r'^-(\d{8})$', (None, 'date')),
-                  (r'^(\d{8})-$', ('date', None)),
-                  (r'^(\d{8})-(\d{8})$', ('date', 'date')),
-                  (r'^-(\d{10})$', (None, 'date')),
-                  (r'^(\d{10})-$', ('date', None)),
-                  (r'^(\d{10})-(\d{10})$', ('date', 'date')),
-                  (r'^-(\d{13})$', (None, 'date')),
-                  (r'^(\d{13})-$', ('date', None)),
-                  (r'^(\d{13})-(\d{13})$', ('date', 'date')),
-                  ]
+        syntax = [
+            (r"^-(\d{8})$", (None, "date")),
+            (r"^(\d{8})-$", ("date", None)),
+            (r"^(\d{8})-(\d{8})$", ("date", "date")),
+            (r"^-(\d{10})$", (None, "date")),
+            (r"^(\d{10})-$", ("date", None)),
+            (r"^(\d{10})-(\d{10})$", ("date", "date")),
+            (r"^-(\d{13})$", (None, "date")),
+            (r"^(\d{13})-$", ("date", None)),
+            (r"^(\d{13})-(\d{13})$", ("date", "date")),
+        ]
         for rex, stype in syntax:
             # Apply the regular expression to text
             match = re.match(rex, text)
@@ -138,9 +149,12 @@ class TimeRange:
                 stop: int = 0
                 if stype[0]:
                     starts = rvals[index]
-                    if stype[0] == 'date' and len(starts) == 8:
-                        start = int(datetime.strptime(starts, '%Y%m%d').replace(
-                            tzinfo=timezone.utc).timestamp())
+                    if stype[0] == "date" and len(starts) == 8:
+                        start = int(
+                            datetime.strptime(starts, "%Y%m%d")
+                            .replace(tzinfo=timezone.utc)
+                            .timestamp()
+                        )
                     elif len(starts) == 13:
                         start = int(starts) // 1000
                     else:
@@ -148,15 +162,19 @@ class TimeRange:
                     index += 1
                 if stype[1]:
                     stops = rvals[index]
-                    if stype[1] == 'date' and len(stops) == 8:
-                        stop = int(datetime.strptime(stops, '%Y%m%d').replace(
-                            tzinfo=timezone.utc).timestamp())
+                    if stype[1] == "date" and len(stops) == 8:
+                        stop = int(
+                            datetime.strptime(stops, "%Y%m%d")
+                            .replace(tzinfo=timezone.utc)
+                            .timestamp()
+                        )
                     elif len(stops) == 13:
                         stop = int(stops) // 1000
                     else:
                         stop = int(stops)
                 if start > stop > 0:
                     raise ConfigurationError(
-                        f'Start date is after stop date for timerange "{text}"')
+                        f'Start date is after stop date for timerange "{text}"'
+                    )
                 return cls(stype[0], stype[1], start, stop)
         raise ConfigurationError(f'Incorrect syntax for timerange "{text}"')

freqtrade/constants.py

@@ -3,110 +3,161 @@
 """
 bot constants
 """
-from typing import Any, Dict, List, Literal, Tuple
 
-from freqtrade.enums import CandleType, PriceType, RPCMessageType
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+from freqtrade.enums import CandleType, PriceType
 
 
 DOCS_LINK = "https://www.freqtrade.io/en/stable"
-DEFAULT_CONFIG = 'config.json'
+DEFAULT_CONFIG = "config.json"
 PROCESS_THROTTLE_SECS = 5  # sec
 HYPEROPT_EPOCH = 100  # epochs
 RETRY_TIMEOUT = 30  # sec
-TIMEOUT_UNITS = ['minutes', 'seconds']
-EXPORT_OPTIONS = ['none', 'trades', 'signals']
-DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
-DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
-UNLIMITED_STAKE_AMOUNT = 'unlimited'
+TIMEOUT_UNITS = ["minutes", "seconds"]
+EXPORT_OPTIONS = ["none", "trades", "signals"]
+DEFAULT_DB_PROD_URL = "sqlite:///tradesv3.sqlite"
+DEFAULT_DB_DRYRUN_URL = "sqlite:///tradesv3.dryrun.sqlite"
+UNLIMITED_STAKE_AMOUNT = "unlimited"
 DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05
-REQUIRED_ORDERTIF = ['entry', 'exit']
-REQUIRED_ORDERTYPES = ['entry', 'exit', 'stoploss', 'stoploss_on_exchange']
-PRICING_SIDES = ['ask', 'bid', 'same', 'other']
-ORDERTYPE_POSSIBILITIES = ['limit', 'market']
-_ORDERTIF_POSSIBILITIES = ['GTC', 'FOK', 'IOC', 'PO']
+REQUIRED_ORDERTIF = ["entry", "exit"]
+REQUIRED_ORDERTYPES = ["entry", "exit", "stoploss", "stoploss_on_exchange"]
+PRICING_SIDES = ["ask", "bid", "same", "other"]
+ORDERTYPE_POSSIBILITIES = ["limit", "market"]
+_ORDERTIF_POSSIBILITIES = ["GTC", "FOK", "IOC", "PO"]
 ORDERTIF_POSSIBILITIES = _ORDERTIF_POSSIBILITIES + [t.lower() for t in _ORDERTIF_POSSIBILITIES]
 STOPLOSS_PRICE_TYPES = [p for p in PriceType]
-HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
-                         'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
-                         'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily',
-                         'CalmarHyperOptLoss',
-                         'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
-                         'ProfitDrawDownHyperOptLoss']
-AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter',
-                       'PerformanceFilter', 'PrecisionFilter', 'PriceFilter',
-                       'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter',
-                       'VolatilityFilter']
-AVAILABLE_PROTECTIONS = ['CooldownPeriod',
-                         'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
-AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']
-BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
-BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
-BACKTEST_CACHE_DEFAULT = 'day'
+HYPEROPT_LOSS_BUILTIN = [
+    "ShortTradeDurHyperOptLoss",
+    "OnlyProfitHyperOptLoss",
+    "SharpeHyperOptLoss",
+    "SharpeHyperOptLossDaily",
+    "SortinoHyperOptLoss",
+    "SortinoHyperOptLossDaily",
+    "CalmarHyperOptLoss",
+    "MaxDrawDownHyperOptLoss",
+    "MaxDrawDownRelativeHyperOptLoss",
+    "ProfitDrawDownHyperOptLoss",
+]
+AVAILABLE_PAIRLISTS = [
+    "StaticPairList",
+    "VolumePairList",
+    "ProducerPairList",
+    "RemotePairList",
+    "MarketCapPairList",
+    "AgeFilter",
+    "FullTradesFilter",
+    "OffsetFilter",
+    "PerformanceFilter",
+    "PrecisionFilter",
+    "PriceFilter",
+    "RangeStabilityFilter",
+    "ShuffleFilter",
+    "SpreadFilter",
+    "VolatilityFilter",
+]
+AVAILABLE_PROTECTIONS = ["CooldownPeriod", "LowProfitPairs", "MaxDrawdown", "StoplossGuard"]
+AVAILABLE_DATAHANDLERS = ["json", "jsongz", "hdf5", "feather", "parquet"]
+BACKTEST_BREAKDOWNS = ["day", "week", "month"]
+BACKTEST_CACHE_AGE = ["none", "day", "week", "month"]
+BACKTEST_CACHE_DEFAULT = "day"
 DRY_RUN_WALLET = 1000
-DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
+DATETIME_PRINT_FORMAT = "%Y-%m-%d %H:%M:%S"
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
-DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
+DEFAULT_DATAFRAME_COLUMNS = ["date", "open", "high", "low", "close", "volume"]
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
-DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
+DEFAULT_TRADES_COLUMNS = ["timestamp", "id", "type", "side", "price", "amount", "cost"]
+DEFAULT_ORDERFLOW_COLUMNS = ["level", "bid", "ask", "delta"]
 TRADES_DTYPES = {
-    'timestamp': 'int64',
-    'id': 'str',
-    'type': 'str',
-    'side': 'str',
-    'price': 'float64',
-    'amount': 'float64',
-    'cost': 'float64',
+    "timestamp": "int64",
+    "id": "str",
+    "type": "str",
+    "side": "str",
+    "price": "float64",
+    "amount": "float64",
+    "cost": "float64",
 }
-TRADING_MODES = ['spot', 'margin', 'futures']
-MARGIN_MODES = ['cross', 'isolated', '']
+TRADING_MODES = ["spot", "margin", "futures"]
+MARGIN_MODES = ["cross", "isolated", ""]
 
-LAST_BT_RESULT_FN = '.last_result.json'
-FTHYPT_FILEVERSION = 'fthypt_fileversion'
+LAST_BT_RESULT_FN = ".last_result.json"
+FTHYPT_FILEVERSION = "fthypt_fileversion"
 
-USERPATH_HYPEROPTS = 'hyperopts'
-USERPATH_STRATEGIES = 'strategies'
-USERPATH_NOTEBOOKS = 'notebooks'
-USERPATH_FREQAIMODELS = 'freqaimodels'
+USERPATH_HYPEROPTS = "hyperopts"
+USERPATH_STRATEGIES = "strategies"
+USERPATH_NOTEBOOKS = "notebooks"
+USERPATH_FREQAIMODELS = "freqaimodels"
 
-TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
-WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw']
+TELEGRAM_SETTING_OPTIONS = ["on", "off", "silent"]
+WEBHOOK_FORMAT_OPTIONS = ["form", "json", "raw"]
 FULL_DATAFRAME_THRESHOLD = 100
 CUSTOM_TAG_MAX_LENGTH = 255
-DL_DATA_TIMEFRAMES = ['1m', '5m']
+DL_DATA_TIMEFRAMES = ["1m", "5m"]
 
-ENV_VAR_PREFIX = 'FREQTRADE__'
+ENV_VAR_PREFIX = "FREQTRADE__"
 
-CANCELED_EXCHANGE_STATES = ('cancelled', 'canceled', 'expired')
-NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ('closed',)
+CANCELED_EXCHANGE_STATES = ("cancelled", "canceled", "expired")
+NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ("closed",)
 
 # Define decimals per coin for outputs
 # Only used for outputs.
 DECIMAL_PER_COIN_FALLBACK = 3  # Should be low to avoid listing all possible FIAT's
 DECIMALS_PER_COIN = {
-    'BTC': 8,
-    'ETH': 5,
+    "BTC": 8,
+    "ETH": 5,
 }
 
-DUST_PER_COIN = {
-    'BTC': 0.0001,
-    'ETH': 0.01
-}
+DUST_PER_COIN = {"BTC": 0.0001, "ETH": 0.01}
 
 # Source files with destination directories within user-directory
 USER_DATA_FILES = {
-    'sample_strategy.py': USERPATH_STRATEGIES,
-    'sample_hyperopt_loss.py': USERPATH_HYPEROPTS,
-    'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS,
+    "sample_strategy.py": USERPATH_STRATEGIES,
+    "sample_hyperopt_loss.py": USERPATH_HYPEROPTS,
+    "strategy_analysis_example.ipynb": USERPATH_NOTEBOOKS,
 }
 
 SUPPORTED_FIAT = [
-    "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK",
-    "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
-    "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
-    "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR",
-    "USD", "BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
+    "AUD",
+    "BRL",
+    "CAD",
+    "CHF",
+    "CLP",
+    "CNY",
+    "CZK",
+    "DKK",
+    "EUR",
+    "GBP",
+    "HKD",
+    "HUF",
+    "IDR",
+    "ILS",
+    "INR",
+    "JPY",
+    "KRW",
+    "MXN",
+    "MYR",
+    "NOK",
+    "NZD",
+    "PHP",
+    "PKR",
+    "PLN",
+    "RUB",
+    "UAH",
+    "SEK",
+    "SGD",
+    "THB",
+    "TRY",
+    "TWD",
+    "ZAR",
+    "USD",
+    "BTC",
+    "ETH",
+    "XRP",
+    "LTC",
+    "BCH",
+    "BNB",
+    "",  # Allow empty field in config.
 ]
 
 MINIMAL_CONFIG = {
@@ -117,574 +168,10 @@ MINIMAL_CONFIG = {
         "key": "",
         "secret": "",
         "pair_whitelist": [],
-        "ccxt_async_config": {
-        }
-    }
-}
-
-__MESSAGE_TYPE_DICT: Dict[str, Dict[str, str]] = {x: {'type': 'object'} for x in RPCMessageType}
-
-# Required json-schema for user specified config
-CONF_SCHEMA = {
-    'type': 'object',
-    'properties': {
-        'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
-        'new_pairs_days': {'type': 'integer', 'default': 30},
-        'timeframe': {'type': 'string'},
-        'stake_currency': {'type': 'string'},
-        'stake_amount': {
-            'type': ['number', 'string'],
-            'minimum': 0.0001,
-            'pattern': UNLIMITED_STAKE_AMOUNT
-        },
-        'tradable_balance_ratio': {
-            'type': 'number',
-            'minimum': 0.0,
-            'maximum': 1,
-            'default': 0.99
-        },
-        'available_capital': {
-            'type': 'number',
-            'minimum': 0,
-        },
-        'amend_last_stake_amount': {'type': 'boolean', 'default': False},
-        'last_stake_amount_min_ratio': {
-            'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5
-        },
-        'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT},
-        'dry_run': {'type': 'boolean'},
-        'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET},
-        'cancel_open_orders_on_exit': {'type': 'boolean', 'default': False},
-        'process_only_new_candles': {'type': 'boolean'},
-        'minimal_roi': {
-            'type': 'object',
-            'patternProperties': {
-                '^[0-9.]+$': {'type': 'number'}
-            },
-        },
-        'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5},
-        'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
-        'trailing_stop': {'type': 'boolean'},
-        'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
-        'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
-        'trailing_only_offset_is_reached': {'type': 'boolean'},
-        'use_exit_signal': {'type': 'boolean'},
-        'exit_profit_only': {'type': 'boolean'},
-        'exit_profit_offset': {'type': 'number'},
-        'ignore_roi_if_entry_signal': {'type': 'boolean'},
-        'ignore_buying_expired_candle_after': {'type': 'number'},
-        'trading_mode': {'type': 'string', 'enum': TRADING_MODES},
-        'margin_mode': {'type': 'string', 'enum': MARGIN_MODES},
-        'reduce_df_footprint': {'type': 'boolean', 'default': False},
-        'minimum_trade_amount': {'type': 'number', 'default': 10},
-        'targeted_trade_amount': {'type': 'number', 'default': 20},
-        'lookahead_analysis_exportfilename': {'type': 'string'},
-        'startup_candle': {
-            'type': 'array',
-            'uniqueItems': True,
-            'default': [199, 399, 499, 999, 1999],
-        },
-        'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99},
-        'backtest_breakdown': {
-            'type': 'array',
-            'items': {'type': 'string', 'enum': BACKTEST_BREAKDOWNS}
-        },
-        'bot_name': {'type': 'string'},
-        'unfilledtimeout': {
-            'type': 'object',
-            'properties': {
-                'entry': {'type': 'number', 'minimum': 1},
-                'exit': {'type': 'number', 'minimum': 1},
-                'exit_timeout_count': {'type': 'number', 'minimum': 0, 'default': 0},
-                'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
-            }
-        },
-        'entry_pricing': {
-            'type': 'object',
-            'properties': {
-                'price_last_balance': {
-                    'type': 'number',
-                    'minimum': 0,
-                    'maximum': 1,
-                    'exclusiveMaximum': False,
-                },
-                'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'},
-                'use_order_book': {'type': 'boolean'},
-                'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
-                'check_depth_of_market': {
-                    'type': 'object',
-                    'properties': {
-                        'enabled': {'type': 'boolean'},
-                        'bids_to_ask_delta': {'type': 'number', 'minimum': 0},
-                    }
-                },
-            },
-            'required': ['price_side']
-        },
-        'exit_pricing': {
-            'type': 'object',
-            'properties': {
-                'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'},
-                'price_last_balance': {
-                    'type': 'number',
-                    'minimum': 0,
-                    'maximum': 1,
-                    'exclusiveMaximum': False,
-                },
-                'use_order_book': {'type': 'boolean'},
-                'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
-            },
-            'required': ['price_side']
-        },
-        'custom_price_max_distance_ratio': {
-            'type': 'number', 'minimum': 0.0
-        },
-        'order_types': {
-            'type': 'object',
-            'properties': {
-                'entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
-                'exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
-                'force_exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
-                'force_entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
-                'emergency_exit': {
-                    'type': 'string',
-                    'enum': ORDERTYPE_POSSIBILITIES,
-                    'default': 'market'},
-                'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
-                'stoploss_on_exchange': {'type': 'boolean'},
-                'stoploss_price_type': {'type': 'string', 'enum': STOPLOSS_PRICE_TYPES},
-                'stoploss_on_exchange_interval': {'type': 'number'},
-                'stoploss_on_exchange_limit_ratio': {'type': 'number', 'minimum': 0.0,
-                                                     'maximum': 1.0}
-            },
-            'required': ['entry', 'exit', 'stoploss', 'stoploss_on_exchange']
-        },
-        'order_time_in_force': {
-            'type': 'object',
-            'properties': {
-                'entry': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES},
-                'exit': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}
-            },
-            'required': REQUIRED_ORDERTIF
-        },
-        'exchange': {'$ref': '#/definitions/exchange'},
-        'edge': {'$ref': '#/definitions/edge'},
-        'freqai': {'$ref': '#/definitions/freqai'},
-        'external_message_consumer': {'$ref': '#/definitions/external_message_consumer'},
-        'experimental': {
-            'type': 'object',
-            'properties': {
-                'block_bad_exchanges': {'type': 'boolean'}
-            }
-        },
-        'pairlists': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS},
-                },
-                'required': ['method'],
-            }
-        },
-        'protections': {
-            'type': 'array',
-            'items': {
-                'type': 'object',
-                'properties': {
-                    'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS},
-                    'stop_duration': {'type': 'number', 'minimum': 0.0},
-                    'stop_duration_candles': {'type': 'number', 'minimum': 0},
-                    'trade_limit': {'type': 'number', 'minimum': 1},
-                    'lookback_period': {'type': 'number', 'minimum': 1},
-                    'lookback_period_candles': {'type': 'number', 'minimum': 1},
-                },
-                'required': ['method'],
-            }
-        },
-        'telegram': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'token': {'type': 'string'},
-                'chat_id': {'type': 'string'},
-                'allow_custom_messages': {'type': 'boolean', 'default': True},
-                'balance_dust_level': {'type': 'number', 'minimum': 0.0},
-                'notification_settings': {
-                    'type': 'object',
-                    'default': {},
-                    'properties': {
-                        'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'entry': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'entry_fill': {
-                            'type': 'string',
-                            'enum': TELEGRAM_SETTING_OPTIONS,
-                            'default': 'off'
-                        },
-                        'entry_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, },
-                        'exit': {
-                            'type': ['string', 'object'],
-                            'additionalProperties': {
-                                'type': 'string',
-                                'enum': TELEGRAM_SETTING_OPTIONS
-                            }
-                        },
-                        'exit_fill': {
-                            'type': 'string',
-                            'enum': TELEGRAM_SETTING_OPTIONS,
-                            'default': 'on'
-                        },
-                        'exit_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'protection_trigger': {
-                            'type': 'string',
-                            'enum': TELEGRAM_SETTING_OPTIONS,
-                            'default': 'on'
-                        },
-                        'protection_trigger_global': {
-                            'type': 'string',
-                            'enum': TELEGRAM_SETTING_OPTIONS,
-                            'default': 'on'
-                        },
-                        'show_candle': {
-                            'type': 'string',
-                            'enum': ['off', 'ohlc'],
-                            'default': 'off'
-                        },
-                        'strategy_msg': {
-                            'type': 'string',
-                            'enum': TELEGRAM_SETTING_OPTIONS,
-                            'default': 'on'
-                        },
-                    }
-                },
-                'reload': {'type': 'boolean'},
-            },
-            'required': ['enabled', 'token', 'chat_id'],
-        },
-        'webhook': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'url': {'type': 'string'},
-                'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'},
-                'retries': {'type': 'integer', 'minimum': 0},
-                'retry_delay': {'type': 'number', 'minimum': 0},
-                **__MESSAGE_TYPE_DICT,
-                # **{x: {'type': 'object'} for x in RPCMessageType},
-                # Below -> Deprecated
-                'webhookentry': {'type': 'object'},
-                'webhookentrycancel': {'type': 'object'},
-                'webhookentryfill': {'type': 'object'},
-                'webhookexit': {'type': 'object'},
-                'webhookexitcancel': {'type': 'object'},
-                'webhookexitfill': {'type': 'object'},
-                'webhookstatus': {'type': 'object'},
-            },
-        },
-        'discord': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'webhook_url': {'type': 'string'},
-                "exit_fill": {
-                    'type': 'array', 'items': {'type': 'object'},
-                    'default': [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Close rate": "{close_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Close date": "{close_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Profit": "{profit_amount} {stake_currency}"},
-                        {"Profitability": "{profit_ratio:.2%}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Exit Reason": "{exit_reason}"},
-                        {"Strategy": "{strategy}"},
-                        {"Timeframe": "{timeframe}"},
-                    ]
-                },
-                "entry_fill": {
-                    'type': 'array', 'items': {'type': 'object'},
-                    'default': [
-                        {"Trade ID": "{trade_id}"},
-                        {"Exchange": "{exchange}"},
-                        {"Pair": "{pair}"},
-                        {"Direction": "{direction}"},
-                        {"Open rate": "{open_rate}"},
-                        {"Amount": "{amount}"},
-                        {"Open date": "{open_date:%Y-%m-%d %H:%M:%S}"},
-                        {"Enter tag": "{enter_tag}"},
-                        {"Strategy": "{strategy} {timeframe}"},
-                    ]
-                },
-            }
-        },
-        'api_server': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'listen_ip_address': {'format': 'ipv4'},
-                'listen_port': {
-                    'type': 'integer',
-                    'minimum': 1024,
-                    'maximum': 65535
-                },
-                'username': {'type': 'string'},
-                'password': {'type': 'string'},
-                'ws_token': {'type': ['string', 'array'], 'items': {'type': 'string'}},
-                'jwt_secret_key': {'type': 'string'},
-                'CORS_origins': {'type': 'array', 'items': {'type': 'string'}},
-                'verbosity': {'type': 'string', 'enum': ['error', 'info']},
-            },
-            'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password']
-        },
-        'db_url': {'type': 'string'},
-        'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
-        'disableparamexport': {'type': 'boolean'},
-        'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
-        'force_entry_enable': {'type': 'boolean'},
-        'disable_dataframe_checks': {'type': 'boolean'},
-        'internals': {
-            'type': 'object',
-            'default': {},
-            'properties': {
-                'process_throttle_secs': {'type': 'integer'},
-                'interval': {'type': 'integer'},
-                'sd_notify': {'type': 'boolean'},
-            }
-        },
-        'dataformat_ohlcv': {
-            'type': 'string',
-            'enum': AVAILABLE_DATAHANDLERS,
-            'default': 'feather'
-        },
-        'dataformat_trades': {
-            'type': 'string',
-            'enum': AVAILABLE_DATAHANDLERS,
-            'default': 'feather'
-        },
-        'position_adjustment_enable': {'type': 'boolean'},
-        'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1},
-    },
-    'definitions': {
-        'exchange': {
-            'type': 'object',
-            'properties': {
-                'name': {'type': 'string'},
-                'key': {'type': 'string', 'default': ''},
-                'secret': {'type': 'string', 'default': ''},
-                'password': {'type': 'string', 'default': ''},
-                'uid': {'type': 'string'},
-                'pair_whitelist': {
-                    'type': 'array',
-                    'items': {
-                        'type': 'string',
-                    },
-                    'uniqueItems': True
-                },
-                'pair_blacklist': {
-                    'type': 'array',
-                    'items': {
-                        'type': 'string',
-                    },
-                    'uniqueItems': True
-                },
-                'unknown_fee_rate': {'type': 'number'},
-                'outdated_offset': {'type': 'integer', 'minimum': 1},
-                'markets_refresh_interval': {'type': 'integer'},
-                'ccxt_config': {'type': 'object'},
-                'ccxt_async_config': {'type': 'object'}
-            },
-            'required': ['name']
-        },
-        'edge': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean'},
-                'process_throttle_secs': {'type': 'integer', 'minimum': 600},
-                'calculate_since_number_of_days': {'type': 'integer'},
-                'allowed_risk': {'type': 'number'},
-                'stoploss_range_min': {'type': 'number'},
-                'stoploss_range_max': {'type': 'number'},
-                'stoploss_range_step': {'type': 'number'},
-                'minimum_winrate': {'type': 'number'},
-                'minimum_expectancy': {'type': 'number'},
-                'min_trade_number': {'type': 'number'},
-                'max_trade_duration_minute': {'type': 'integer'},
-                'remove_pumps': {'type': 'boolean'}
-            },
-            'required': ['process_throttle_secs', 'allowed_risk']
-        },
-        'external_message_consumer': {
-            'type': 'object',
-            'properties': {
-                'enabled': {'type': 'boolean', 'default': False},
-                'producers': {
-                    'type': 'array',
-                    'items': {
-                        'type': 'object',
-                        'properties': {
-                            'name': {'type': 'string'},
-                            'host': {'type': 'string'},
-                            'port': {
-                                'type': 'integer',
-                                'default': 8080,
-                                'minimum': 0,
-                                'maximum': 65535
-                            },
-                            'secure': {'type': 'boolean', 'default': False},
-                            'ws_token': {'type': 'string'},
-                        },
-                        'required': ['name', 'host', 'ws_token']
-                    }
-                },
-                'wait_timeout': {'type': 'integer', 'minimum': 0},
-                'sleep_time': {'type': 'integer', 'minimum': 0},
-                'ping_timeout': {'type': 'integer', 'minimum': 0},
-                'remove_entry_exit_signals': {'type': 'boolean', 'default': False},
-                'initial_candle_limit': {
-                    'type': 'integer',
-                    'minimum': 0,
-                    'maximum': 1500,
-                    'default': 1500
-                },
-                'message_size_limit': {  # In megabytes
-                    'type': 'integer',
-                    'minimum': 1,
-                    'maxmium': 20,
-                    'default': 8,
-                }
-            },
-            'required': ['producers']
-        },
-        "freqai": {
-            "type": "object",
-            "properties": {
-                "enabled": {"type": "boolean", "default": False},
-                "keras": {"type": "boolean", "default": False},
-                "write_metrics_to_disk": {"type": "boolean", "default": False},
-                "purge_old_models": {"type": ["boolean", "number"], "default": 2},
-                "conv_width": {"type": "integer", "default": 1},
-                "train_period_days": {"type": "integer", "default": 0},
-                "backtest_period_days": {"type": "number", "default": 7},
-                "identifier": {"type": "string", "default": "example"},
-                "feature_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "include_corr_pairlist": {"type": "array"},
-                        "include_timeframes": {"type": "array"},
-                        "label_period_candles": {"type": "integer"},
-                        "include_shifted_candles": {"type": "integer", "default": 0},
-                        "DI_threshold": {"type": "number", "default": 0},
-                        "weight_factor": {"type": "number", "default": 0},
-                        "principal_component_analysis": {"type": "boolean", "default": False},
-                        "use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
-                        "plot_feature_importances": {"type": "integer", "default": 0},
-                        "svm_params": {"type": "object",
-                                       "properties": {
-                                           "shuffle": {"type": "boolean", "default": False},
-                                           "nu": {"type": "number", "default": 0.1}
-                                           },
-                                       },
-                        "shuffle_after_split": {"type": "boolean", "default": False},
-                        "buffer_train_data_candles": {"type": "integer", "default": 0}
-                    },
-                    "required": ["include_timeframes", "include_corr_pairlist", ]
-                },
-                "data_split_parameters": {
-                    "type": "object",
-                    "properties": {
-                        "test_size": {"type": "number"},
-                        "random_state": {"type": "integer"},
-                        "shuffle": {"type": "boolean", "default": False}
-                    },
-                },
-                "model_training_parameters": {
-                    "type": "object"
-                },
-                "rl_config": {
-                    "type": "object",
-                    "properties": {
-                        "drop_ohlc_from_features": {"type": "boolean", "default": False},
-                        "train_cycles": {"type": "integer"},
-                        "max_trade_duration_candles": {"type": "integer"},
-                        "add_state_info": {"type": "boolean", "default": False},
-                        "max_training_drawdown_pct": {"type": "number", "default": 0.02},
-                        "cpu_count": {"type": "integer", "default": 1},
-                        "model_type": {"type": "string", "default": "PPO"},
-                        "policy_type": {"type": "string", "default": "MlpPolicy"},
-                        "net_arch": {"type": "array", "default": [128, 128]},
-                        "randomize_starting_position": {"type": "boolean", "default": False},
-                        "progress_bar": {"type": "boolean", "default": True},
-                        "model_reward_parameters": {
-                            "type": "object",
-                            "properties": {
-                                "rr": {"type": "number", "default": 1},
-                                "profit_aim": {"type": "number", "default": 0.025}
-                            }
-                        }
-                    },
-                },
-            },
-            "required": [
-                "enabled",
-                "train_period_days",
-                "backtest_period_days",
-                "identifier",
-                "feature_parameters",
-                "data_split_parameters"
-            ]
-        },
+        "ccxt_async_config": {},
     },
 }
 
-SCHEMA_TRADE_REQUIRED = [
-    'exchange',
-    'timeframe',
-    'max_open_trades',
-    'stake_currency',
-    'stake_amount',
-    'tradable_balance_ratio',
-    'last_stake_amount_min_ratio',
-    'dry_run',
-    'dry_run_wallet',
-    'exit_pricing',
-    'entry_pricing',
-    'stoploss',
-    'minimal_roi',
-    'internals',
-    'dataformat_ohlcv',
-    'dataformat_trades',
-]
-
-SCHEMA_BACKTEST_REQUIRED = [
-    'exchange',
-    'stake_currency',
-    'stake_amount',
-    'dry_run_wallet',
-    'dataformat_ohlcv',
-    'dataformat_trades',
-]
-SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [
-    'stoploss',
-    'minimal_roi',
-    'max_open_trades'
-]
-
-SCHEMA_MINIMAL_REQUIRED = [
-    'exchange',
-    'dry_run',
-    'dataformat_ohlcv',
-    'dataformat_trades',
-]
-SCHEMA_MINIMAL_WEBSERVER = SCHEMA_MINIMAL_REQUIRED + [
-    'api_server',
-]
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
@@ -696,7 +183,7 @@ CANCEL_REASON = {
     "FORCE_EXIT": "forcesold",
     "REPLACE": "cancelled to be replaced by new limit order",
     "REPLACE_FAILED": "failed to replace order, deleting Trade",
-    "USER_CANCEL": "user requested order cancel"
+    "USER_CANCEL": "user requested order cancel",
 }
 
 # List of pairs with their timeframes
@@ -705,13 +192,16 @@ ListPairsWithTimeframes = List[PairWithTimeframe]
 
 # Type for trades list
 TradeList = List[List]
+# ticks, pair, timeframe, CandleType
+TickWithTimeframe = Tuple[str, str, CandleType, Optional[int], Optional[int]]
+ListTicksWithTimeframes = List[TickWithTimeframe]
 
-LongShort = Literal['long', 'short']
-EntryExit = Literal['entry', 'exit']
-BuySell = Literal['buy', 'sell']
-MakerTaker = Literal['maker', 'taker']
-BidAsk = Literal['bid', 'ask']
-OBLiteral = Literal['asks', 'bids']
+LongShort = Literal["long", "short"]
+EntryExit = Literal["entry", "exit"]
+BuySell = Literal["buy", "sell"]
+MakerTaker = Literal["maker", "taker"]
+BidAsk = Literal["bid", "ask"]
+OBLiteral = Literal["asks", "bids"]
 
 Config = Dict[str, Any]
 # Exchange part of the configuration.
@@ -719,4 +209,4 @@ ExchangeConfig = Dict[str, Any]
 IntOrInf = float
 
 
-EntryExecuteMode = Literal['initial', 'pos_adjust', 'replace']
+EntryExecuteMode = Literal["initial", "pos_adjust", "replace"]

freqtrade/data/__init__.py

@@ -2,7 +2,8 @@
 Module to handle data operations for freqtrade
 """
 
+from freqtrade.data import converter
+
+
 # limit what's imported when using `from freqtrade.data import *`
-__all__ = [
-    'converter'
-]
+__all__ = ["converter"]

freqtrade/data/btanalysis.py

@@ -1,6 +1,7 @@
 """
 Helpers when analyzing backtest data
 """
+
 import logging
 from copy import copy
 from datetime import datetime, timezone
@@ -21,14 +22,35 @@ from freqtrade.types import BacktestHistoryEntryType, BacktestResultType
 logger = logging.getLogger(__name__)
 
 # Newest format
-BT_DATA_COLUMNS = ['pair', 'stake_amount', 'max_stake_amount', 'amount',
-                   'open_date', 'close_date', 'open_rate', 'close_rate',
-                   'fee_open', 'fee_close', 'trade_duration',
-                   'profit_ratio', 'profit_abs', 'exit_reason',
-                   'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
-                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag',
-                   'leverage', 'is_short', 'open_timestamp', 'close_timestamp', 'orders'
-                   ]
+BT_DATA_COLUMNS = [
+    "pair",
+    "stake_amount",
+    "max_stake_amount",
+    "amount",
+    "open_date",
+    "close_date",
+    "open_rate",
+    "close_rate",
+    "fee_open",
+    "fee_close",
+    "trade_duration",
+    "profit_ratio",
+    "profit_abs",
+    "exit_reason",
+    "initial_stop_loss_abs",
+    "initial_stop_loss_ratio",
+    "stop_loss_abs",
+    "stop_loss_ratio",
+    "min_rate",
+    "max_rate",
+    "is_open",
+    "enter_tag",
+    "leverage",
+    "is_short",
+    "open_timestamp",
+    "close_timestamp",
+    "orders",
+]
 
 
 def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
@@ -50,15 +72,16 @@ def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> s
 
     if not filename.is_file():
         raise ValueError(
-            f"Directory '{directory}' does not seem to contain backtest statistics yet.")
+            f"Directory '{directory}' does not seem to contain backtest statistics yet."
+        )
 
     with filename.open() as file:
         data = json_load(file)
 
-    if f'latest_{variant}' not in data:
+    if f"latest_{variant}" not in data:
         raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.")
 
-    return data[f'latest_{variant}']
+    return data[f"latest_{variant}"]
 
 
 def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
@@ -71,7 +94,7 @@ def get_latest_backtest_filename(directory: Union[Path, str]) -> str:
         * `directory/.last_result.json` does not exist
         * `directory/.last_result.json` has the wrong content
     """
-    return get_latest_optimize_filename(directory, 'backtest')
+    return get_latest_optimize_filename(directory, "backtest")
 
 
 def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str:
@@ -85,14 +108,15 @@ def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str:
         * `directory/.last_result.json` has the wrong content
     """
     try:
-        return get_latest_optimize_filename(directory, 'hyperopt')
+        return get_latest_optimize_filename(directory, "hyperopt")
     except ValueError:
         # Return default (legacy) pickle filename
-        return 'hyperopt_results.pickle'
+        return "hyperopt_results.pickle"
 
 
 def get_latest_hyperopt_file(
-        directory: Union[Path, str], predef_filename: Optional[str] = None) -> Path:
+    directory: Union[Path, str], predef_filename: Optional[str] = None
+) -> Path:
     """
     Get latest hyperopt export based on '.last_result.json'.
     :param directory: Directory to search for last result
@@ -107,7 +131,8 @@ def get_latest_hyperopt_file(
     if predef_filename:
         if Path(predef_filename).is_absolute():
             raise ConfigurationError(
-                "--hyperopt-filename expects only the filename, not an absolute path.")
+                "--hyperopt-filename expects only the filename, not an absolute path."
+            )
         return directory / predef_filename
     return directory / get_latest_hyperopt_filename(directory)
 
@@ -126,7 +151,7 @@ def load_backtest_metadata(filename: Union[Path, str]) -> Dict[str, Any]:
     except FileNotFoundError:
         return {}
     except Exception as e:
-        raise OperationalException('Unexpected error while loading backtest metadata.') from e
+        raise OperationalException("Unexpected error while loading backtest metadata.") from e
 
 
 def load_backtest_stats(filename: Union[Path, str]) -> BacktestResultType:
@@ -147,7 +172,7 @@ def load_backtest_stats(filename: Union[Path, str]) -> BacktestResultType:
 
     # Legacy list format does not contain metadata.
     if isinstance(data, dict):
-        data['metadata'] = load_backtest_metadata(filename)
+        data["metadata"] = load_backtest_metadata(filename)
     return data
 
 
@@ -159,38 +184,39 @@ def load_and_merge_backtest_result(strategy_name: str, filename: Path, results:
     :param results: dict to merge the result to.
     """
     bt_data = load_backtest_stats(filename)
-    k: Literal['metadata', 'strategy']
-    for k in ('metadata', 'strategy'):  # type: ignore
+    k: Literal["metadata", "strategy"]
+    for k in ("metadata", "strategy"):
         results[k][strategy_name] = bt_data[k][strategy_name]
-    results['metadata'][strategy_name]['filename'] = filename.stem
-    comparison = bt_data['strategy_comparison']
+    results["metadata"][strategy_name]["filename"] = filename.stem
+    comparison = bt_data["strategy_comparison"]
     for i in range(len(comparison)):
-        if comparison[i]['key'] == strategy_name:
-            results['strategy_comparison'].append(comparison[i])
+        if comparison[i]["key"] == strategy_name:
+            results["strategy_comparison"].append(comparison[i])
             break
 
 
 def _get_backtest_files(dirname: Path) -> List[Path]:
     # Weird glob expression here avoids including .meta.json files.
-    return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
+    return list(reversed(sorted(dirname.glob("backtest-result-*-[0-9][0-9].json"))))
 
 
 def _extract_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
     metadata = load_backtest_metadata(filename)
     return [
         {
-            'filename': filename.stem,
-            'strategy': s,
-            'run_id': v['run_id'],
-            'notes': v.get('notes', ''),
+            "filename": filename.stem,
+            "strategy": s,
+            "run_id": v["run_id"],
+            "notes": v.get("notes", ""),
             # Backtest "run" time
-            'backtest_start_time': v['backtest_start_time'],
+            "backtest_start_time": v["backtest_start_time"],
             # Backtest timerange
-            'backtest_start_ts': v.get('backtest_start_ts', None),
-            'backtest_end_ts': v.get('backtest_end_ts', None),
-            'timeframe': v.get('timeframe', None),
-            'timeframe_detail': v.get('timeframe_detail', None),
-        } for s, v in metadata.items()
+            "backtest_start_ts": v.get("backtest_start_ts", None),
+            "backtest_end_ts": v.get("backtest_end_ts", None),
+            "timeframe": v.get("timeframe", None),
+            "timeframe_detail": v.get("timeframe_detail", None),
+        }
+        for s, v in metadata.items()
     ]
 
 
@@ -218,7 +244,7 @@ def delete_backtest_result(file_abs: Path):
     """
     # *.meta.json
     logger.info(f"Deleting backtest result file: {file_abs.name}")
-    file_abs_meta = file_abs.with_suffix('.meta.json')
+    file_abs_meta = file_abs.with_suffix(".meta.json")
     file_abs.unlink()
     file_abs_meta.unlink()
 
@@ -244,12 +270,13 @@ def get_backtest_market_change(filename: Path, include_ts: bool = True) -> pd.Da
     """
     df = pd.read_feather(filename)
     if include_ts:
-        df.loc[:, '__date_ts'] = df.loc[:, 'date'].astype(np.int64) // 1000 // 1000
+        df.loc[:, "__date_ts"] = df.loc[:, "date"].astype(np.int64) // 1000 // 1000
     return df
 
 
-def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
-                                 min_backtest_date: Optional[datetime] = None) -> Dict[str, Any]:
+def find_existing_backtest_stats(
+    dirname: Union[Path, str], run_ids: Dict[str, str], min_backtest_date: Optional[datetime] = None
+) -> Dict[str, Any]:
     """
     Find existing backtest stats that match specified run IDs and load them.
     :param dirname: pathlib.Path object, or string pointing to the file.
@@ -261,9 +288,9 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
     run_ids = copy(run_ids)
     dirname = Path(dirname)
     results: Dict[str, Any] = {
-        'metadata': {},
-        'strategy': {},
-        'strategy_comparison': [],
+        "metadata": {},
+        "strategy": {},
+        "strategy_comparison": [],
     }
 
     for filename in _get_backtest_files(dirname):
@@ -280,14 +307,14 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
                 continue
 
             if min_backtest_date is not None:
-                backtest_date = strategy_metadata['backtest_start_time']
+                backtest_date = strategy_metadata["backtest_start_time"]
                 backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc)
                 if backtest_date < min_backtest_date:
                     # Do not use a cached result for this strategy as first result is too old.
                     del run_ids[strategy_name]
                     continue
 
-            if strategy_metadata['run_id'] == run_id:
+            if strategy_metadata["run_id"] == run_id:
                 del run_ids[strategy_name]
                 load_and_merge_backtest_result(strategy_name, filename, results)
 
@@ -300,20 +327,20 @@ def _load_backtest_data_df_compatibility(df: pd.DataFrame) -> pd.DataFrame:
     """
     Compatibility support for older backtest data.
     """
-    df['open_date'] = pd.to_datetime(df['open_date'], utc=True)
-    df['close_date'] = pd.to_datetime(df['close_date'], utc=True)
+    df["open_date"] = pd.to_datetime(df["open_date"], utc=True)
+    df["close_date"] = pd.to_datetime(df["close_date"], utc=True)
     # Compatibility support for pre short Columns
-    if 'is_short' not in df.columns:
-        df['is_short'] = False
-    if 'leverage' not in df.columns:
-        df['leverage'] = 1.0
-    if 'enter_tag' not in df.columns:
-        df['enter_tag'] = df['buy_tag']
-        df = df.drop(['buy_tag'], axis=1)
-    if 'max_stake_amount' not in df.columns:
-        df['max_stake_amount'] = df['stake_amount']
-    if 'orders' not in df.columns:
-        df['orders'] = None
+    if "is_short" not in df.columns:
+        df["is_short"] = False
+    if "leverage" not in df.columns:
+        df["leverage"] = 1.0
+    if "enter_tag" not in df.columns:
+        df["enter_tag"] = df["buy_tag"]
+        df = df.drop(["buy_tag"], axis=1)
+    if "max_stake_amount" not in df.columns:
+        df["max_stake_amount"] = df["stake_amount"]
+    if "orders" not in df.columns:
+        df["orders"] = None
     return df
 
 
@@ -329,23 +356,25 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
     data = load_backtest_stats(filename)
     if not isinstance(data, list):
         # new, nested format
-        if 'strategy' not in data:
+        if "strategy" not in data:
             raise ValueError("Unknown dataformat.")
 
         if not strategy:
-            if len(data['strategy']) == 1:
-                strategy = list(data['strategy'].keys())[0]
+            if len(data["strategy"]) == 1:
+                strategy = list(data["strategy"].keys())[0]
             else:
-                raise ValueError("Detected backtest result with more than one strategy. "
-                                 "Please specify a strategy.")
+                raise ValueError(
+                    "Detected backtest result with more than one strategy. "
+                    "Please specify a strategy."
+                )
 
-        if strategy not in data['strategy']:
+        if strategy not in data["strategy"]:
             raise ValueError(
                 f"Strategy {strategy} not available in the backtest result. "
                 f"Available strategies are '{','.join(data['strategy'].keys())}'"
-                )
+            )
 
-        data = data['strategy'][strategy]['trades']
+        data = data["strategy"][strategy]["trades"]
         df = pd.DataFrame(data)
         if not df.empty:
             df = _load_backtest_data_df_compatibility(df)
@@ -353,7 +382,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
     else:
         # old format - only with lists.
         raise OperationalException(
-            "Backtest-results with only trades data are no longer supported.")
+            "Backtest-results with only trades data are no longer supported."
+        )
     if not df.empty:
         df = df.sort_values("open_date").reset_index(drop=True)
     return df
@@ -368,23 +398,26 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     :return: dataframe with open-counts per time-period in timeframe
     """
     from freqtrade.exchange import timeframe_to_resample_freq
+
     timeframe_freq = timeframe_to_resample_freq(timeframe)
-    dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
-                                     freq=timeframe_freq))
-             for row in results[['open_date', 'close_date']].iterrows()]
+    dates = [
+        pd.Series(pd.date_range(row[1]["open_date"], row[1]["close_date"], freq=timeframe_freq))
+        for row in results[["open_date", "close_date"]].iterrows()
+    ]
     deltas = [len(x) for x in dates]
-    dates = pd.Series(pd.concat(dates).values, name='date')
+    dates = pd.Series(pd.concat(dates).values, name="date")
     df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
 
     df2 = pd.concat([dates, df2], axis=1)
-    df2 = df2.set_index('date')
-    df_final = df2.resample(timeframe_freq)[['pair']].count()
-    df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
+    df2 = df2.set_index("date")
+    df_final = df2.resample(timeframe_freq)[["pair"]].count()
+    df_final = df_final.rename({"pair": "open_trades"}, axis=1)
     return df_final
 
 
-def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
-                          max_open_trades: IntOrInf) -> pd.DataFrame:
+def evaluate_result_multi(
+    results: pd.DataFrame, timeframe: str, max_open_trades: IntOrInf
+) -> pd.DataFrame:
     """
     Find overlapping trades by expanding each trade once per period it was open
     and then counting overlaps
@@ -394,7 +427,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     :return: dataframe with open-counts per time-period in freq
     """
     df_final = analyze_trade_parallelism(results, timeframe)
-    return df_final[df_final['open_trades'] > max_open_trades]
+    return df_final[df_final["open_trades"] > max_open_trades]
 
 
 def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.DataFrame:
@@ -405,9 +438,9 @@ def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.
     """
     df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS)
     if len(df) > 0:
-        df['close_date'] = pd.to_datetime(df['close_date'], utc=True)
-        df['open_date'] = pd.to_datetime(df['open_date'], utc=True)
-        df['close_rate'] = df['close_rate'].astype('float64')
+        df["close_date"] = pd.to_datetime(df["close_date"], utc=True)
+        df["open_date"] = pd.to_datetime(df["open_date"], utc=True)
+        df["close_rate"] = df["close_rate"].astype("float64")
     return df
 
 
@@ -429,8 +462,13 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
     return trades
 
 
-def load_trades(source: str, db_url: str, exportfilename: Path,
-                no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame:
+def load_trades(
+    source: str,
+    db_url: str,
+    exportfilename: Path,
+    no_trades: bool = False,
+    strategy: Optional[str] = None,
+) -> pd.DataFrame:
     """
     Based on configuration option 'trade_source':
     * loads data from DB (using `db_url`)
@@ -451,8 +489,9 @@ def load_trades(source: str, db_url: str, exportfilename: Path,
         return load_backtest_data(exportfilename, strategy)
 
 
-def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
-                             date_index=False) -> pd.DataFrame:
+def extract_trades_of_period(
+    dataframe: pd.DataFrame, trades: pd.DataFrame, date_index=False
+) -> pd.DataFrame:
     """
     Compare trades and backtested pair DataFrames to get trades performed on backtested period
     :return: the DataFrame of a trades of period
@@ -461,8 +500,9 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
         trades_start = dataframe.index[0]
         trades_stop = dataframe.index[-1]
     else:
-        trades_start = dataframe.iloc[0]['date']
-        trades_stop = dataframe.iloc[-1]['date']
-    trades = trades.loc[(trades['open_date'] >= trades_start) &
-                        (trades['close_date'] <= trades_stop)]
+        trades_start = dataframe.iloc[0]["date"]
+        trades_stop = dataframe.iloc[-1]["date"]
+    trades = trades.loc[
+        (trades["open_date"] >= trades_start) & (trades["close_date"] <= trades_stop)
+    ]
     return trades

freqtrade/data/converter/__init__.py

@@ -1,28 +1,40 @@
-from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format,
-                                                ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
-                                                order_book_to_dataframe, reduce_dataframe_footprint,
-                                                trim_dataframe, trim_dataframes)
-from freqtrade.data.converter.trade_converter import (convert_trades_format,
-                                                      convert_trades_to_ohlcv, trades_convert_types,
-                                                      trades_df_remove_duplicates,
-                                                      trades_dict_to_list, trades_list_to_df,
-                                                      trades_to_ohlcv)
+from freqtrade.data.converter.converter import (
+    clean_ohlcv_dataframe,
+    convert_ohlcv_format,
+    ohlcv_fill_up_missing_data,
+    ohlcv_to_dataframe,
+    order_book_to_dataframe,
+    reduce_dataframe_footprint,
+    trim_dataframe,
+    trim_dataframes,
+)
+from freqtrade.data.converter.orderflow import populate_dataframe_with_trades
+from freqtrade.data.converter.trade_converter import (
+    convert_trades_format,
+    convert_trades_to_ohlcv,
+    trades_convert_types,
+    trades_df_remove_duplicates,
+    trades_dict_to_list,
+    trades_list_to_df,
+    trades_to_ohlcv,
+)
 
 
 __all__ = [
-    'clean_ohlcv_dataframe',
-    'convert_ohlcv_format',
-    'ohlcv_fill_up_missing_data',
-    'ohlcv_to_dataframe',
-    'order_book_to_dataframe',
-    'reduce_dataframe_footprint',
-    'trim_dataframe',
-    'trim_dataframes',
-    'convert_trades_format',
-    'convert_trades_to_ohlcv',
-    'trades_convert_types',
-    'trades_df_remove_duplicates',
-    'trades_dict_to_list',
-    'trades_list_to_df',
-    'trades_to_ohlcv',
+    "clean_ohlcv_dataframe",
+    "convert_ohlcv_format",
+    "ohlcv_fill_up_missing_data",
+    "ohlcv_to_dataframe",
+    "order_book_to_dataframe",
+    "reduce_dataframe_footprint",
+    "trim_dataframe",
+    "trim_dataframes",
+    "convert_trades_format",
+    "convert_trades_to_ohlcv",
+    "populate_dataframe_with_trades",
+    "trades_convert_types",
+    "trades_df_remove_duplicates",
+    "trades_dict_to_list",
+    "trades_list_to_df",
+    "trades_to_ohlcv",
 ]