Merge pull request #9876 from freqtrade/new_release

New release 2024.2

.github/dependabot.yml

@@ -10,8 +10,17 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
+    time: "03:00"
+    timezone: "Etc/UTC"
   open-pull-requests-limit: 15
   target-branch: develop
+  groups:
+    types:
+      patterns:
+        - "types-*"
+    pytest:
+      patterns:
+        - "pytest*"
 
 - package-ecosystem: "github-actions"
   directory: "/"

.github/workflows/ci.yml

@@ -25,26 +25,25 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
-      if: runner.os == 'Linux'
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -55,7 +54,6 @@ jobs:
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
     - name: Installation - *nix
-      if: runner.os == 'Linux'
       run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
@@ -90,7 +88,7 @@ jobs:
 
     - name: Backtesting (multi)
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade new-strategy -s AwesomeStrategy
         freqtrade new-strategy -s AwesomeStrategyMin --template minimal
@@ -98,7 +96,7 @@ jobs:
 
     - name: Hyperopt
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
@@ -108,7 +106,7 @@ jobs:
 
     - name: Run Ruff
       run: |
-        ruff check --format=github .
+        ruff check --output-format=github .
 
     - name: Mypy
       run: |
@@ -122,35 +120,34 @@ jobs:
           details: Freqtrade CI failed on ${{ matrix.os }}
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  build_macos:
+  build-macos:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest ]
-        python-version: ["3.9", "3.10", "3.11"]
+        os: [ "macos-latest", "macos-13" ]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         check-latest: true
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
-        key: ${{ runner.os }}-dependencies
+        key: ${{ matrix.os }}-dependencies
 
     - name: pip cache (macOS)
-      uses: actions/cache@v3
-      if: runner.os == 'macOS'
+      uses: actions/cache@v4
       with:
         path: ~/Library/Caches/pip
-        key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
+        key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
 
     - name: TA binary *nix
       if: steps.cache.outputs.cache-hit != 'true'
@@ -158,7 +155,6 @@ jobs:
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
     - name: Installation - macOS
-      if: runner.os == 'macOS'
       run: |
         # brew update
         # TODO: Should be the brew upgrade
@@ -166,16 +162,21 @@ jobs:
         # https://github.com/actions/runner-images/issues/6817
         rm /usr/local/bin/2to3 || true
         rm /usr/local/bin/2to3-3.11 || true
+        rm /usr/local/bin/2to3-3.12 || true
         rm /usr/local/bin/idle3 || true
         rm /usr/local/bin/idle3.11 || true
+        rm /usr/local/bin/idle3.12 || true
         rm /usr/local/bin/pydoc3 || true
         rm /usr/local/bin/pydoc3.11 || true
+        rm /usr/local/bin/pydoc3.12 || true
         rm /usr/local/bin/python3 || true
         rm /usr/local/bin/python3.11 || true
+        rm /usr/local/bin/python3.12 || true
         rm /usr/local/bin/python3-config || true
         rm /usr/local/bin/python3.11-config || true
+        rm /usr/local/bin/python3.12-config || true
 
-        brew install hdf5 c-blosc
+        brew install hdf5 c-blosc libomp
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
         export TA_LIBRARY_PATH=${HOME}/dependencies/lib
@@ -200,14 +201,14 @@ jobs:
 
     - name: Backtesting
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade new-strategy -s AwesomeStrategyAdv --template advanced
         freqtrade backtesting --datadir tests/testdata --strategy AwesomeStrategyAdv
 
     - name: Hyperopt
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
@@ -217,7 +218,7 @@ jobs:
 
     - name: Run Ruff
       run: |
-        ruff check --format=github .
+        ruff check --output-format=github .
 
     - name: Mypy
       run: |
@@ -231,24 +232,24 @@ jobs:
           details: Test Succeeded!
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  build_windows:
+  build-windows:
 
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Pip cache (Windows)
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~\AppData\Local\pip\Cache
         key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -275,19 +276,19 @@ jobs:
 
     - name: Backtesting
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy
 
     - name: Hyperopt
       run: |
-        cp config_examples/config_bittrex.example.json config.json
+        cp tests/testdata/config.tests.json config.json
         freqtrade create-userdir --userdir user_data
         freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all
 
     - name: Run Ruff
       run: |
-        ruff check --format=github .
+        ruff check --output-format=github .
 
     - name: Mypy
       run: |
@@ -301,13 +302,13 @@ jobs:
           details: Test Failed
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  mypy_version_check:
+  mypy-version-check:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
 
@@ -321,12 +322,12 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
 
-  docs_check:
+  docs-check:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
@@ -336,7 +337,7 @@ jobs:
         ./tests/test_docs.sh
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11"
 
@@ -362,20 +363,19 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.9"
+        python-version: "3.11"
 
     - name: Cache_dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: ~/dependencies/
         key: ${{ runner.os }}-dependencies
 
     - name: pip cache (linux)
-      uses: actions/cache@v3
-      if: runner.os == 'Linux'
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip
@@ -386,7 +386,6 @@ jobs:
         cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd ..
 
     - name: Installation - *nix
-      if: runner.os == 'Linux'
       run: |
         python -m pip install --upgrade pip wheel
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
@@ -399,17 +398,17 @@ jobs:
       env:
         CI_WEB_PROXY: http://152.67.78.211:13128
       run: |
-        pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun
+        pytest --random-order --longrun --durations 20 -n auto --dist loadscope
 
 
   # Notify only once - when CI completes (and after deploy) in case it's successfull
   notify-complete:
     needs: [
       build_linux,
-      build_macos,
-      build_windows,
-      docs_check,
-      mypy_version_check,
+      build-macos,
+      build-windows,
+      docs-check,
+      mypy-version-check,
       pre-commit,
       build_linux_online
     ]
@@ -436,8 +435,63 @@ jobs:
           details: Test Completed!
           webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
 
-  deploy:
-    needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check, pre-commit ]
+  build:
+    name: "Build"
+    needs: [ build_linux, build-macos, build-windows, docs-check, mypy-version-check, pre-commit ]
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Build distribution
+      run: |
+        pip install -U build
+        python -m build --sdist --wheel
+
+    - name: Upload artifacts 📦
+      uses: actions/upload-artifact@v4
+      with:
+        name: freqtrade-build
+        path: |
+          dist
+        retention-days: 10
+
+  deploy-pypi:
+    name: "Deploy to PyPI"
+    needs: [ build ]
+    runs-on: ubuntu-22.04
+    if: (github.event_name == 'release')
+    environment:
+      name: release
+      url: https://pypi.org/p/freqtrade
+    permissions:
+      id-token: write
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download artifact  📦
+      uses: actions/download-artifact@v4
+      with:
+        name: freqtrade-build
+        path: dist
+
+    - name: Publish to PyPI (Test)
+      uses: pypa/gh-action-pypi-publish@v1.8.11
+      with:
+        repository-url: https://test.pypi.org/legacy/
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@v1.8.11
+
+
+  deploy-docker:
+    needs: [ build_linux, build-macos, build-windows, docs-check, mypy-version-check, pre-commit ]
     runs-on: ubuntu-22.04
 
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
@@ -446,34 +500,15 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11"
 
     - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
-      id: extract_branch
-
-    - name: Build distribution
+      id: extract-branch
       run: |
-        pip install -U setuptools wheel
-        python setup.py sdist bdist_wheel
-
-    - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.10
-      if: (github.event_name == 'release')
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_test_password }}
-        repository_url: https://test.pypi.org/legacy/
-
-    - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.10
-      if: (github.event_name == 'release')
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+        echo "GITHUB_REF='${GITHUB_REF}'"
+        echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT"
 
     - name: Dockerhub login
       env:
@@ -502,14 +537,15 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }}
       run: |
         build_helpers/publish_docker_multi.sh
 
-  deploy_arm:
+  deploy-arm:
+    name: "Deploy Docker"
     permissions:
       packages: write
-    needs: [ deploy ]
+    needs: [ deploy-docker ]
     # Only run on 64bit machines
     runs-on: [self-hosted, linux, ARM64]
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
@@ -518,9 +554,10 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})"
-      id: extract_branch
+      id: extract-branch
+      run: |
+        echo "GITHUB_REF='${GITHUB_REF}'"
+        echo "branch=${GITHUB_REF##*/}" >> "$GITHUB_OUTPUT"
 
     - name: Dockerhub login
       env:
@@ -531,7 +568,7 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        BRANCH_NAME: ${{ steps.extract-branch.outputs.branch }}
         GHCR_USERNAME: ${{ github.actor }}
         GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |

.github/workflows/docker_update_readme.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Docker Hub Description
-      uses: peter-evans/dockerhub-description@v3
+      uses: peter-evans/dockerhub-description@v4
       env:
         DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/pre-commit-update.yml

@@ -0,0 +1,45 @@
+name: Pre-commit auto-update
+
+on:
+  # every day at midnight
+  schedule:
+    - cron: "0 3 * * 2"
+  # on demand
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run auto-update
+        run: pre-commit autoupdate
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
+      - uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.REPO_SCOPED_TOKEN }}
+          add-paths: .pre-commit-config.yaml
+          labels: |
+            Tech maintenance
+            Dependencies
+          branch: update/pre-commit-hooks
+          title: Update pre-commit hooks
+          commit-message: "chore: update pre-commit hooks"
+          committer: Freqtrade Bot <noreply@github.com>
+          body: Update versions of pre-commit hooks to latest version.
+          delete-branch: true

.gitignore

@@ -111,7 +111,6 @@ target/
 #exceptions
 !*.gitkeep
 !config_examples/config_binance.example.json
-!config_examples/config_bittrex.example.json
 !config_examples/config_full.example.json
 !config_examples/config_kraken.example.json
 !config_examples/config_freqai.example.json

.pre-commit-config.yaml

@@ -2,27 +2,28 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pycqa/flake8
-    rev: "6.0.0"
+    rev: "7.0.0"
     hooks:
       - id: flake8
+        additional_dependencies: [Flake8-pyproject]
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.5.1"
+    rev: "v1.8.0"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
-          - types-cachetools==5.3.0.6
+          - types-cachetools==5.3.0.7
           - types-filelock==3.2.7
-          - types-requests==2.31.0.4
-          - types-tabulate==0.9.0.3
-          - types-python-dateutil==2.8.19.14
-          - SQLAlchemy==2.0.21
+          - types-requests==2.31.0.20240218
+          - types-tabulate==0.9.0.20240106
+          - types-python-dateutil==2.8.19.20240106
+          - SQLAlchemy==2.0.27
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
-    rev: "5.12.0"
+    rev: "5.13.2"
     hooks:
       - id: isort
         name: isort (python)
@@ -30,12 +31,12 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.270'
+    rev: 'v0.2.2'
     hooks:
       - id: ruff
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: end-of-file-fixer
         exclude: |

CONTRIBUTING.md

@@ -125,7 +125,7 @@ Exceptions:
 
 Contributors may be given commit privileges. Preference will be given to those with:
 
-1. Past contributions to Freqtrade and other related open-source projects. Contributions to Freqtrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered.
+1. Past contributions to Freqtrade and other related open-source projects. Contributions to Freqtrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Both quantity and quality are considered.
 1. A coding style that the other core committers find simple, minimal, and clean.
 1. Access to resources for cross-platform development and testing.
 1. Time to devote to the project regularly.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.5-slim-bullseye as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

MANIFEST.in

@@ -5,3 +5,5 @@ recursive-include freqtrade/templates/ *.j2 *.ipynb
 include freqtrade/exchange/binance_leverage_tiers.json
 include freqtrade/rpc/api_server/ui/fallback_file.html
 include freqtrade/rpc/api_server/ui/favicon.ico
+
+prune tests

README.md

@@ -28,9 +28,9 @@ hesitate to read the source code and understand the mechanism of this bot.
 Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
 - [X] [Binance](https://www.binance.com/)
-- [X] [Bittrex](https://bittrex.com/)
+- [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

build_helpers/publish_docker_arm64.sh

@@ -54,7 +54,7 @@ docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM
 docker tag freqtrade:$TAG_FREQAI_RL_ARM ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM
 
 # Run backtest
-docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
+docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
 
 if [ $? -ne 0 ]; then
     echo "failed running backtest"

build_helpers/publish_docker_multi.sh

@@ -67,7 +67,7 @@ docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI
 docker tag freqtrade:$TAG_FREQAI_RL ${CACHE_IMAGE}:$TAG_FREQAI_RL
 
 # Run backtest
-docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
+docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
 
 if [ $? -ne 0 ]; then
     echo "failed running backtest"

config_examples/config_binance.example.json

@@ -1,6 +1,6 @@
 {
     "max_open_trades": 3,
-    "stake_currency": "BTC",
+    "stake_currency": "USDT",
     "stake_amount": 0.05,
     "tradable_balance_ratio": 0.99,
     "fiat_display_currency": "USD",
@@ -36,21 +36,21 @@
         "ccxt_async_config": {
         },
         "pair_whitelist": [
-            "ALGO/BTC",
-            "ATOM/BTC",
-            "BAT/BTC",
-            "BCH/BTC",
-            "BRD/BTC",
-            "EOS/BTC",
-            "ETH/BTC",
-            "IOTA/BTC",
-            "LINK/BTC",
-            "LTC/BTC",
-            "NEO/BTC",
-            "NXS/BTC",
-            "XMR/BTC",
-            "XRP/BTC",
-            "XTZ/BTC"
+            "ALGO/USDT",
+            "ATOM/USDT",
+            "BAT/USDT",
+            "BCH/USDT",
+            "BRD/USDT",
+            "EOS/USDT",
+            "ETH/USDT",
+            "IOTA/USDT",
+            "LINK/USDT",
+            "LTC/USDT",
+            "NEO/USDT",
+            "NXS/USDT",
+            "XMR/USDT",
+            "XRP/USDT",
+            "XTZ/USDT"
         ],
         "pair_blacklist": [
             "BNB/.*"

config_examples/config_freqai.example.json

@@ -52,7 +52,7 @@
         "train_period_days": 15,
         "backtest_period_days": 7,
         "live_retrain_hours": 0,
-        "identifier": "uniqe-id",
+        "identifier": "unique-id",
         "feature_parameters": {
             "include_timeframes": [
                 "3m",

docker/Dockerfile.armhf

@@ -1,4 +1,4 @@
-FROM python:3.9.16-slim-bullseye as base
+FROM python:3.11.8-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8
@@ -11,12 +11,13 @@ ENV FT_APP_ENV="docker"
 # Prepare environment
 RUN mkdir /freqtrade \
   && apt-get update \
-  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \
+  && apt-get -y install sudo libatlas3-base libopenblas-dev curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \
   && apt-get clean \
   && useradd -u 1000 -G sudo -U -m ftuser \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
-  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers \
+  && pip install --upgrade pip
 
 WORKDIR /freqtrade
 
@@ -25,20 +26,16 @@ FROM base as python-deps
 RUN  apt-get update \
   && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \
   && apt-get clean \
-  && pip install --upgrade pip \
   && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf
 
 # Install TA-lib
 COPY build_helpers/* /tmp/
-RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
-ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy==1.25.2 \
-  && pip install --user /tmp/pyarrow-*.whl \
-  && pip install --user --no-build-isolation TA-Lib==0.4.28 \
+RUN  pip install --user --no-cache-dir numpy \
+  && pip install --user --no-index --find-links /tmp/ pyarrow TA-Lib==0.4.28 \
   && pip install --user --no-cache-dir -r requirements.txt
 
 # Copy dependencies to runtime-image

docker/Dockerfile.jupyter

@@ -1,8 +1,8 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-# Pin jupyter-client to avoid tornado version conflict
-RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir
+# Pin prompt-toolkit to avoid questionary version conflict
+RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -6,7 +6,7 @@ services:
       context: ..
       dockerfile: docker/Dockerfile.jupyter
     restart: unless-stopped
-    container_name: freqtrade
+    # container_name: freqtrade
     ports:
       - "127.0.0.1:8888:8888"
     volumes:

docs/backtesting.md

@@ -31,9 +31,9 @@ optional arguments:
                         Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
-  --data-format-ohlcv {json,jsongz,hdf5}
+  --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
+                        (default: `feather`).
   --max-open-trades INT
                         Override the value of the `max_open_trades`
                         configuration setting.
@@ -170,11 +170,11 @@ freqtrade backtesting --strategy AwesomeStrategy --dry-run-wallet 1000
 
 Using a different on-disk historical candle (OHLCV) data source
 
-Assume you downloaded the history data from the Bittrex exchange and kept it in the `user_data/data/bittrex-20180101` directory. 
+Assume you downloaded the history data from the Binance exchange and kept it in the `user_data/data/binance-20180101` directory. 
 You can then use this data for backtesting as follows:
 
 ```bash
-freqtrade backtesting --strategy AwesomeStrategy --datadir user_data/data/bittrex-20180101 
+freqtrade backtesting --strategy AwesomeStrategy --datadir user_data/data/binance-20180101 
 ```
 
 ---
@@ -618,13 +618,13 @@ To compare multiple strategies, a list of Strategies can be provided to backtest
 This is limited to 1 timeframe value per run. However, data is only loaded once from disk so if you have multiple
 strategies you'd like to compare, this will give a nice runtime boost.
 
-All listed Strategies need to be in the same directory.
+All listed Strategies need to be in the same directory, unless also `--recursive-strategy-search` is specified, where sub-directories within the strategy directory are also considered.
 
 ``` bash
 freqtrade backtesting --timerange 20180401-20180410 --timeframe 5m --strategy-list Strategy001 Strategy002 --export trades
 ```
 
-This will save the results to `user_data/backtest_results/backtest-result-<strategy>.json`, injecting the strategy-name into the target filename.
+This will save the results to `user_data/backtest_results/backtest-result-<datetime>.json`, including results for both `Strategy001` and `Strategy002`.
 There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table).
 Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy.
 

docs/configuration.md

@@ -14,7 +14,7 @@ You can specify a different configuration file used by the bot with the `-c/--co
 If you used the [Quick start](docker_quickstart.md#docker-quick-start) method for installing
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
 
-If the default configuration file is not created we recommend to use `freqtrade new-config --config config.json` to generate a basic configuration file.
+If the default configuration file is not created we recommend to use `freqtrade new-config --config user_data/config.json` to generate a basic configuration file.
 
 The Freqtrade configuration file is to be written in JSON format.
 
@@ -321,7 +321,7 @@ For example, if you have 10 ETH available in your wallet on the exchange and `tr
 To fully utilize compounding profits when using multiple bots on the same exchange account, you'll want to limit each bot to a certain starting balance.
 This can be accomplished by setting `available_capital` to the desired starting balance.
 
-Assuming your account has 10.000 USDT and you want to run 2 different strategies on this exchange.
+Assuming your account has 10000 USDT and you want to run 2 different strategies on this exchange.
 You'd set `available_capital=5000` - granting each bot an initial capital of 5000 USDT.
 The bot will then split this starting balance equally into `max_open_trades` buckets.
 Profitable trades will result in increased stake-sizes for this bot - without affecting the stake-sizes of the other bot.
@@ -572,9 +572,11 @@ In addition to fiat currencies, a range of crypto currencies is supported.
 The valid values are:
 
 ```json
-"BTC", "ETH", "XRP", "LTC", "BCH", "USDT"
+"BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ```
 
+Removing `fiat_display_currency` completely from the configuration will skip initializing coingecko, and will not show any FIAT currency conversion. This has no importance for the correct functioning of the bot.
+
 ## Using Dry-run mode
 
 We recommend starting the bot in the Dry-run mode to see how your bot will
@@ -594,7 +596,7 @@ creating trades on the exchange.
 
 ```json
 "exchange": {
-    "name": "bittrex",
+    "name": "binance",
     "key": "key",
     "secret": "secret",
     ...
@@ -644,7 +646,7 @@ API Keys are usually only required for live trading (trading for real money, bot
 ```json
 {
     "exchange": {
-        "name": "bittrex",
+        "name": "binance",
         "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b",
         "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5",
         //"password": "", // Optional, not needed by all exchanges)

docs/developer.md

@@ -318,6 +318,7 @@ Additional tests / steps to complete:
 * Check if balance shows correctly (*)
 * Create market order (*)
 * Create limit order (*)
+* Cancel order (*)
 * Complete trade (enter + exit) (*)
   * Compare result calculation between exchange and bot
   * Ensure fees are applied correctly (check the database against the exchange)
@@ -418,6 +419,9 @@ This part of the documentation is aimed at maintainers, and shows how to create
 
 ### Create release branch
 
+!!! Note
+    Make sure that the `stable` branch is up-to-date!
+
 First, pick a commit that's about one week old (to not include latest additions to releases).
 
 ``` bash
@@ -430,14 +434,11 @@ Determine if crucial bugfixes have been made between this commit and the current
 * Merge the release branch (stable) into this branch.
 * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi.
 * Commit this part.
-* push that branch to the remote and create a PR against the stable branch.
+* Push that branch to the remote and create a PR against the **stable branch**.
 * Update develop version to next version following the pattern `2019.8-dev`.
 
 ### Create changelog from git commits
 
-!!! Note
-    Make sure that the `stable` branch is up-to-date!
-
 ``` bash
 # Needs to be done before merging / pulling that branch.
 git log --oneline --no-decorate --no-merges stable..new_release

docs/exchanges.md

@@ -127,6 +127,8 @@ Freqtrade will not attempt to change these settings.
 
 ## Kraken
 
+Kraken supports [time_in_force](configuration.md#understand-order_time_in_force) with settings "GTC" (good till cancelled), "IOC" (immediate-or-cancel) and "PO" (Post only) settings.
+
 !!! Tip "Stoploss on Exchange"
     Kraken supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
     You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.
@@ -136,6 +138,43 @@ Freqtrade will not attempt to change these settings.
 The Kraken API does only provide 720 historic candles, which is sufficient for Freqtrade dry-run and live trade modes, but is a problem for backtesting.
 To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data.
 
+To speed up downloading, you can download the [trades zip files](https://support.kraken.com/hc/en-us/articles/360047543791-Downloadable-historical-market-data-time-and-sales-) kraken provides.
+These are usually updated once per quarter. Freqtrade expects these files to be placed in `user_data/data/kraken/trades_csv`.
+
+A structure as follows can make sense if using incremental files, with the "full" history in one directory, and incremental files in different directories.
+The assumption for this mode is that the data is downloaded and unzipped keeping filenames as they are.
+Duplicate content will be ignored (based on timestamp) - though the assumption is that there is no gap in the data.
+
+This means, if your "full" history ends in Q4 2022 - then both incremental updates Q1 2023 and Q2 2023 are available.
+Not having this will lead to incomplete data, and therefore invalid results while using the data.
+
+```
+└── trades_csv
+    ├── Kraken_full_history
+    │   ├── BCHEUR.csv
+    │   └── XBTEUR.csv
+    ├── Kraken_Trading_History_Q1_2023
+    │   ├── BCHEUR.csv
+    │   └── XBTEUR.csv
+    └── Kraken_Trading_History_Q2_2023
+        ├── BCHEUR.csv
+        └── XBTEUR.csv
+```
+
+You can convert these files into freqtrade files:
+
+``` bash
+freqtrade convert-trade-data --exchange kraken --format-from kraken_csv --format-to feather
+# Convert trade data to different ohlcv timeframes
+freqtrade trades-to-ohlcv -p BTC/EUR BCH/EUR --exchange kraken -t 1m 5m 15m 1h
+```
+
+The converted data also makes downloading data possible, and will start the download after the latest loaded trade.
+
+``` bash
+freqtrade download-data --exchange kraken --dl-trades -p BTC/EUR BCH/EUR 
+```
+
 !!! Warning "Downloading data from kraken"
     Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine.
     It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient.
@@ -144,48 +183,6 @@ To download data for the Kraken exchange, using `--dl-trades` is mandatory, othe
     Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate.
     So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased.
 
-## Bittrex
-
-### Order types
-
-Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy).
-
-Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment.
-Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected.
-
-### Volume pairlist
-
-Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume.
-
-Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode).
-
-### Restricted markets
-
-Bittrex split its exchange into US and International versions.
-The International version has more pairs available, however the API always returns all pairs, so there is currently no automated way to detect if you're affected by the restriction.
-
-If you have restricted pairs in your whitelist, you'll get a warning message in the log on Freqtrade startup for each restricted pair.
-
-The warning message will look similar to the following:
-
-``` output
-[...] Message: bittrex {"success":false,"message":"RESTRICTED_MARKET","result":null,"explanation":null}"
-```
-
-If you're an "International" customer on the Bittrex exchange, then this warning will probably not impact you.
-If you're a US customer, the bot will fail to create orders for these pairs, and you should remove them from your whitelist.
-
-You can get a list of restricted markets by using the following snippet:
-
-``` python
-import ccxt
-ct = ccxt.bittrex()
-lm = ct.load_markets()
-
-res = [p for p, x in lm.items() if 'US' in x['info']['prohibitedIn']]
-print(res)
-```
-
 ## Kucoin
 
 Kucoin requires a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
@@ -211,10 +208,10 @@ Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force)
 For Kucoin, it is suggested to add `"KCS/<STAKE>"` to your blacklist to avoid issues, unless you are willing to maintain enough extra `KCS` on the account or unless you're willing to disable using `KCS` for fees. 
 Kucoin accounts may use `KCS` for fees, and if a trade happens to be on `KCS`, further trades may consume this position and make the initial `KCS` trade unsellable as the expected amount is not there anymore.
 
-## Huobi
+## HTX (formerly Huobi)
 
 !!! Tip "Stoploss on Exchange"
-    Huobi supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
+    HTX supports `stoploss_on_exchange` and uses `stop-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
 
 ## OKX (former OKEX)
 
@@ -265,6 +262,24 @@ We do strongly recommend to limit all API keys to the IP you're going to use it
     Bybit (futures only) supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.
     On futures, Bybit supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use.
 
+## Bitmart
+
+Bitmart requires the API key Memo (the name you give the API key) to go along with the exchange key and secret.
+It's therefore required to pass the UID as well.
+
+```json
+"exchange": {
+    "name": "bitmart",
+    "uid": "your_bitmart_api_key_memo",
+    "secret": "your_exchange_secret",
+    "password": "your_exchange_api_key_password",
+    // ...
+}
+```
+
+!!! Warning "Necessary Verification"
+    Bitmart requires Verification Lvl2 to successfully trade on the spot market through the API - even though trading via UI works just fine with just Lvl1 verification.
+
 ## All exchanges
 
 Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys.

docs/faq.md

@@ -128,15 +128,9 @@ This warning can point to one of the below problems:
 * Barely traded pair -> Check the pair on the exchange webpage, look at the timeframe your strategy uses. If the pair does not have any volume in some candles (usually visualized with a "volume 0" bar, and a "_" as candle), this pair did not have any trades in this timeframe. These pairs should ideally be avoided, as they can cause problems with order-filling.
 * API problem -> API returns wrong data (this only here for completeness, and should not happen with supported exchanges).
 
-### I'm getting the "RESTRICTED_MARKET" message in the log
-
-Currently known to happen for US Bittrex users.
-
-Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information.
-
 ### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy
 
-As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io).
+As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Gate.io).
 
 To fix this, redefine order types in the strategy to use "limit" instead of "market":
 

docs/freqai-configuration.md

@@ -162,7 +162,8 @@ Below are the values you can expect to include/use inside a typical strategy dat
 | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float.
 | `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2.
 | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
-| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
+| `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%` (see details below). <br> **Datatype:** Depends on the feature created by the user.
+| `df['%%*']` | Any dataframe column prepended with `%%` in `feature_engineering_*()` is treated as a training feature, just the same as the above `%` prepend. However, in this case, the features are returned back to the strategy for FreqUI/plot-dataframe plotting and monitoring in Dry/Live/Backtesting <br> **Datatype:** Depends on the feature created by the user. Please note that features created in `feature_engineering_expand()` will have automatic FreqAI naming schemas depending on the expansions that you configured (i.e. `include_timeframes`, `include_corr_pairlist`, `indicators_periods_candles`, `include_shifted_candles`). So if you want to plot `%%-rsi` from `feature_engineering_expand_all()`, the final naming scheme for your plotting config would be: `%%-rsi-period_10_ETH/USDT:USDT_1h` for the `rsi` feature with `period=10`, `timeframe=1h`, and `pair=ETH/USDT:USDT` (the `:USDT` is added if you are using futures pairs). It is useful to simply add `print(dataframe.columns)` in your `populate_indicators()` after `self.freqai.start()` to see the full list of available features that are returned to the strategy for plotting purposes.
 
 ## Setting the `startup_candle_count`
 

docs/freqai-feature-engineering.md

@@ -7,7 +7,7 @@ Low level feature engineering is performed in the user strategy within a set of
 |  Function | Description |
 |---------------|-------------|
 | `feature_engineering_expand_all()` | This optional function will automatically expand the defined features on the config defined `indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
-| `feature_engineering_expand_basic()` | This optional function will automatically expand the defined features on the config defined `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`. Note: this function does *not* expand across `include_periods_candles`.
+| `feature_engineering_expand_basic()` | This optional function will automatically expand the defined features on the config defined `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`. Note: this function does *not* expand across `indicator_periods_candles`.
 | `feature_engineering_standard()` | This optional function will be called once with the dataframe of the base timeframe. This is the final function to be called, which means that the dataframe entering this function will contain all the features and columns from the base asset created by the other `feature_engineering_expand` functions. This function is a good place to do custom exotic feature extractions (e.g. tsfresh). This function is also a good place for any feature that should not be auto-expanded upon (e.g., day of the week).
 | `set_freqai_targets()` | Required function to set the targets for the model. All targets must be prepended with `&` to be recognized by the FreqAI internals.
 

docs/freqai-parameter-table.md

@@ -74,7 +74,6 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 |  |  **Reinforcement Learning Parameters within the `freqai.rl_config` sub dictionary**
 | `rl_config` | A dictionary containing the control parameters for a Reinforcement Learning model. <br> **Datatype:** Dictionary.
 | `train_cycles` | Training time steps will be set based on the `train_cycles * number of training data points. <br> **Datatype:** Integer.
-| `cpu_count` | Number of processors to dedicate to the Reinforcement Learning training process. <br> **Datatype:** int.
 | `max_trade_duration_candles`| Guides the agent training to keep trades below desired length. Example usage shown in `prediction_models/ReinforcementLearner.py` within the customizable `calculate_reward()` function. <br> **Datatype:** int.
 | `model_type` | Model string from stable_baselines3 or SBcontrib. Available strings include: `'TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'PPO', 'A2C', 'DQN'`. User should ensure that `model_training_parameters` match those available to the corresponding stable_baselines3 model by visiting their documentaiton. [PPO doc](https://stable-baselines3.readthedocs.io/en/master/modules/ppo.html) (external website) <br> **Datatype:** string.
 | `policy_type` | One of the available policy types from stable_baselines3 <br> **Datatype:** string.

docs/freqai-running.md

@@ -41,11 +41,11 @@ FreqAI stores new model files after each successful training. These files become
 
 ```json
     "freqai": {
-        "purge_old_models": true,
+        "purge_old_models": 4,
     }
 ```
 
-This will automatically purge all models older than the two most recently trained ones to save disk space.
+This will automatically purge all models older than the four most recently trained ones to save disk space. Inputing "0" will never purge any models.
 
 ## Backtesting
 
@@ -68,7 +68,7 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
     This way, you can return to using any model you wish by simply specifying the `identifier`.
 
 !!! Note
-    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire backtest timerange. This means that you should be sure that features do look-ahead into the future.
+    Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire training timerange. This means that you should be sure that features do not look-ahead into the future.
     More details about look-ahead bias can be found in [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies).
 
 ---

docs/freqai.md

@@ -114,6 +114,11 @@ Here we compile some external materials that provide deeper looks into various c
 - [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495)
 - [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665)
 
+
+## Support
+
+You can find support for FreqAI in a variety of places, including the [Freqtrade discord](https://discord.gg/Jd8JYeWHc4), the dedicated [FreqAI discord](https://discord.gg/7AMWACmbjT), and in [github issues](https://github.com/freqtrade/freqtrade/issues).
+
 ## Credits
 
 FreqAI is developed by a group of individuals who all contribute specific skillsets to the project.

docs/hyperopt.md

@@ -337,11 +337,15 @@ There are four parameter types each suited for different purposes.
 * `CategoricalParameter` - defines a parameter with a predetermined number of choices.
 * `BooleanParameter` - Shorthand for `CategoricalParameter([True, False])` - great for "enable" parameters.
 
-!!! Tip "Disabling parameter optimization"
-    Each parameter takes two boolean parameters:
-    * `load` - when set to `False` it will not load values configured in `buy_params` and `sell_params`.
-    * `optimize` - when set to `False` parameter will not be included in optimization process.
-    Use these parameters to quickly prototype various ideas.
+### Parameter options
+
+There are two parameter options that can help you to quickly test various ideas:
+
+* `optimize` - when set to `False`, the parameter will not be included in optimization process. (Default: True)
+* `load` - when set to `False`, results of a previous hyperopt run (in `buy_params` and `sell_params` either in your strategy or the JSON output file) will not be used as the starting value for subsequent hyperopts. The default value specified in the parameter will be used instead. (Default: True)
+
+!!! Tip "Effects of `load=False` on backtesting"
+    Be aware that setting the `load` option to `False` will mean backtesting will also use the default value specified in the parameter and *not* the value found through hyperoptimisation.
 
 !!! Warning
     Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
@@ -435,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s
 ??? Hint "Performance tip"
     During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage
 
-    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality.
+    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality.
     * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value.
 
     These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
@@ -922,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should
 
 To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
 
-Should results not match, please double-check to make sure you transferred all conditions correctly.
-Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy.
-You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+### Why do my backtest results not match my hyperopt results?
+Should results not match, check the following factors:
+
+* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter).
+* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly.
+* Check the logs to verify what parameters are being set and what values are being used.
+* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy.
+* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting.

docs/includes/pairlists.md

@@ -6,7 +6,7 @@ In your configuration, you can use Static Pairlist (defined by the [`StaticPairL
 
 Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
-If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
+If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler.
 
 Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist.
 
@@ -24,6 +24,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`VolumePairList`](#volume-pair-list)
 * [`ProducerPairList`](#producerpairlist)
 * [`RemotePairList`](#remotepairlist)
+* [`MarketCapPairList`](#marketcappairlist)
 * [`AgeFilter`](#agefilter)
 * [`FullTradesFilter`](#fulltradesfilter)
 * [`OffsetFilter`](#offsetfilter)
@@ -67,7 +68,7 @@ When used in the leading position of the chain of Pairlist Handlers, the `pair_w
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
 The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
-Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
+Filtering instances (not the first position in the list) will not apply any cache (beyond caching candles for the duration of the candle in advanced mode) and will always use up-to-date data.
 
 `VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -112,8 +113,8 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
 !!! Warning "Performance implications when using lookback range"
     If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
 
-??? Tip "Unsupported exchanges (Bittrex, Gemini)"
-    On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume.
+??? Tip "Unsupported exchanges"
+    On some exchanges (like Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume.
     To roughly simulate 24h volume, you can use the following configuration.
     Please note that These pairlists will only refresh once per day.
 
@@ -192,14 +193,15 @@ The RemotePairList is defined in the pairlists section of the configuration sett
         "refresh_period": 1800,
         "keep_pairlist_on_failure": true,
         "read_timeout": 60,
-        "bearer_token": "my-bearer-token"
+        "bearer_token": "my-bearer-token",
+        "save_to_file": "user_data/filename.json" 
     }
 ]
 ```
 
 The optional `mode` option specifies if the pairlist should be used as a `blacklist` or as a `whitelist`. The default value is "whitelist".
 
-The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append".
+The optional `processing_mode` option in the RemotePairList configuration determines how the retrieved pairlist is processed. It can have two values: "filter" or "append". The default value is "filter".
 
 In "filter" mode, the retrieved pairlist is used as a filter. Only the pairs present in both the original pairlist and the retrieved pairlist are included in the final pairlist. Other pairs are filtered out.
 
@@ -207,6 +209,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All
 
 The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
 
+The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file.
+
+??? Example "Multi bot with shared pairlist example"
+    
+    `save_to_file` can be used to save the pairlist to a file with Bot1:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "https://example.com/pairlist",
+            "number_assets": 10,
+            "refresh_period": 1800,
+            "keep_pairlist_on_failure": true,
+            "read_timeout": 60,
+            "save_to_file": "user_data/filename.json" 
+        }
+    ]
+    ```
+
+    This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration:
+
+    ```json
+    "pairlists": [
+        {
+            "method": "RemotePairList",
+            "mode": "whitelist",
+            "pairlist_url": "file:///user_data/filename.json",
+            "number_assets": 10,
+            "refresh_period": 10,
+            "keep_pairlist_on_failure": true,
+        }
+    ]
+    ```    
+
 The user is responsible for providing a server or local file that returns a JSON object with the following structure:
 
 ```json
@@ -227,6 +265,25 @@ The optional `bearer_token` will be included in the requests Authorization Heade
 !!! Note
     In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned.
 
+#### MarketCapPairList
+
+`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks.
+
+```json
+"pairlists": [
+    {
+        "method": "MarketCapPairList",
+        "number_assets": 20,
+        "max_rank": 50,
+        "refresh_period": 86400
+    }
+]
+```
+
+`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.
+
+`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list).
+
 #### AgeFilter
 
 Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity).

docs/includes/showcase.md

@@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community.
 - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/)
 - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical).
 - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc).
-- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc).
+- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc).
 - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston).
 - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston).
 - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects.

docs/index.md

@@ -40,9 +40,9 @@ Freqtrade is a free and open source crypto trading bot written in Python. It is
 Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange.
 
 - [X] [Binance](https://www.binance.com/)
-- [X] [Bittrex](https://bittrex.com/)
+- [X] [Bitmart](https://bitmart.com/)
 - [X] [Gate.io](https://www.gate.io/ref/6266643)
-- [X] [Huobi](http://huobi.com/)
+- [X] [HTX](https://www.htx.com/) (Former Huobi)
 - [X] [Kraken](https://kraken.com/)
 - [X] [OKX](https://okx.com/) (Former OKEX)
 - [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_

docs/recursive-analysis.md

@@ -40,11 +40,41 @@ usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                     [--startup-candle STARTUP_CANDLES [STARTUP_CANDLES ...]]
 
 optional arguments:
--p PAIR, --pairs PAIR
+  -h, --help            show this help message and exit
+  -i TIMEFRAME, --timeframe TIMEFRAME
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
+  --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `feather`).
+  -p PAIR, --pairs PAIR
                         Limit command to this pair.
---startup-candle STARTUP_CANDLE [STARTUP_CANDLE ...]
+  --startup-candle STARTUP_CANDLE [STARTUP_CANDLE ...]
                         Provide a space-separated list of startup_candle_count to
                         be checked. Default : `199 399 499 999 1999`.
+
+Common arguments:
+  -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
+  --logfile FILE        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
+  -V, --version         show program's version number and exit
+  -c PATH, --config PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH
+                        Path to directory with historical backtesting data.
+  --userdir PATH, --user-data-dir PATH
+                        Path to userdata directory.
+
+Strategy arguments:
+  -s NAME, --strategy NAME
+                        Specify strategy class name which will be used by the
+                        bot.
+  --strategy-path PATH  Specify additional strategy lookup path.
+  --timerange TIMERANGE
+                        Specify what timerange of data to use.
 ```
 
 ### Why are odd-numbered default startup candles used?

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
-markdown==3.4.4
+markdown==3.5.2
 mkdocs==1.5.3
-mkdocs-material==9.4.1
+mkdocs-material==9.5.11
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.3
-jinja2==3.1.2
+pymdown-extensions==10.7
+jinja2==3.1.3

docs/rest-api.md

@@ -134,13 +134,16 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 | `reload_config` | Reloads the configuration file.
 | `trades` | List last trades. Limited to 500 trades per call.
 | `trade/<tradeid>` | Get specific trade.
-| `trade/<tradeid>` | DELETE - Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
-| `trade/<tradeid>/open-order` | DELETE - Cancel open order for this trade.
-| `trade/<tradeid>/reload` | GET - Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange.
+| `trades/<tradeid>` | DELETE - Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
+| `trades/<tradeid>/open-order` | DELETE - Cancel open order for this trade.
+| `trades/<tradeid>/reload` | GET - Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange.
 | `show_config` | Shows part of the current configuration with relevant settings to operation.
 | `logs` | Shows last log messages.
 | `status` | Lists all open trades.
 | `count` | Displays number of trades used and available.
+| `entries [pair]` | Shows profit statistics for each enter tags for given pair (or all pairs if pair isn't given). Pair is optional.
+| `exits [pair]` | Shows profit statistics for each exit reasons for given pair (or all pairs if pair isn't given). Pair is optional.
+| `mix_tags [pair]` | Shows profit statistics for each combinations of enter tag + exit reasons for given pair (or all pairs if pair isn't given). Pair is optional.
 | `locks` | Displays currently locked pairs.
 | `delete_lock <lock_id>` | Deletes (disables) the lock by id.
 | `profit` | Display a summary of your profit/loss from close trades and some stats about your performance.

docs/sql_cheatsheet.md

@@ -109,7 +109,7 @@ Freqtrade does not depend or install any additional database driver. Please refe
 The following systems have been tested and are known to work with freqtrade:
 
 * sqlite (default)
-* PostgreSQL)
+* PostgreSQL
 * MariaDB
 
 !!! Warning

docs/stoploss.md

@@ -30,7 +30,7 @@ The Order-type will be ignored if only one mode is available.
 |----------|-------------|
 | Binance  | limit |
 | Binance Futures  | market, limit |
-| Huobi    | limit |
+| HTX (former Huobi)    | limit |
 | kraken   | market, limit |
 | Gate     | limit |
 | Okx      | limit |

docs/strategy-callbacks.md

@@ -489,7 +489,7 @@ The helper function `stoploss_from_absolute()` can be used to convert from an ab
             dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
             trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
             candle = dataframe.iloc[-1].squeeze()
-            sign = 1 if trade.is_short else -1
+            side = 1 if trade.is_short else -1
             return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), 
                                           current_rate, is_short=trade.is_short,
                                           leverage=trade.leverage)
@@ -760,26 +760,38 @@ The `position_adjustment_enable` strategy property enables the usage of `adjust_
 For performance reasons, it's disabled by default and freqtrade will show a warning message on startup if enabled.
 `adjust_trade_position()` can be used to perform additional orders, for example to manage risk with DCA (Dollar Cost Averaging) or to increase or decrease positions.
 
-`max_entry_position_adjustment` property is used to limit the number of additional buys per trade (on top of the first buy) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment buys.
-
-The strategy is expected to return a stake_amount (in stake currency) between `min_stake` and `max_stake` if and when an additional buy order should be made (position is increased).
-If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
 Additional orders also result in additional fees and those orders don't count towards `max_open_trades`.
 
 This callback is **not** called when there is an open order (either buy or sell) waiting for execution.
 
 `adjust_trade_position()` is called very frequently for the duration of a trade, so you must keep your implementation as performant as possible.
 
-Additional Buys are ignored once you have reached the maximum amount of extra buys that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
+Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade.
+Adjustment orders can be assigned with a tag by returning a 2 element Tuple, with the first element being the adjustment amount, and the 2nd element the tag (e.g. `return 250, 'increase_favorable_conditions'`).
 
-Position adjustments will always be applied in the direction of the trade, so a positive value will always increase your position (negative values will decrease your position), no matter if it's a long or short trade. Modifications to leverage are not possible, and the stake-amount is assumed to be before applying leverage.
+Modifications to leverage are not possible, and the stake-amount returned is assumed to be before applying leverage.
+
+### Increase position
+
+The strategy is expected to return a positive **stake_amount** (in stake currency) between `min_stake` and `max_stake` if and when an additional entry order should be made (position is increased -> buy order for long trades, sell order for short trades).
+
+If there are not enough funds in the wallet (the return value is above `max_stake`) then the signal will be ignored.
+`max_entry_position_adjustment` property is used to limit the number of additional entries per trade (on top of the first entry order) that the bot can execute. By default, the value is -1 which means the bot have no limit on number of adjustment entries.
+
+Additional entries are ignored once you have reached the maximum amount of extra entries that you have set on `max_entry_position_adjustment`, but the callback is called anyway looking for partial exits.
+
+### Decrease position
+
+The strategy is expected to return a negative stake_amount (in stake currency) for a partial exit.
+Returning the full owned stake at that point (based on the current price) (`-(trade.amount / trade.leverage) * current_exit_rate`) results in a full exit.  
+Returning a value more than the above (so remaining stake_amount would become negative) will result in the bot ignoring the signal.
 
 !!! Note "About stake size"
     Using fixed stake size means it will be the amount used for the first order, just like without position adjustment.
     If you wish to buy additional orders with DCA, then make sure to leave enough funds in the wallet for that.
     Using 'unlimited' stake amount with DCA orders requires you to also implement the `custom_stake_amount()` callback to avoid allocating all funds to the initial order.
 
-!!! Warning
+!!! Warning "Stoploss calculation"
     Stoploss is still calculated from the initial opening price, not averaged price.
     Regular stoploss rules still apply (cannot move down).
 
@@ -789,6 +801,11 @@ Position adjustments will always be applied in the direction of the trade, so a
     During backtesting this callback is called for each candle in `timeframe` or `timeframe_detail`, so run-time performance will be affected.
     This can also cause deviating results between live and backtesting, since backtesting can adjust the trade only once per candle, whereas live could adjust the trade multiple times per candle.
 
+!!! Warning "Performance with many position adjustments"
+    Position adjustments can be a good approach to increase a strategy's output - but it can also have drawbacks if using this feature extensively.  
+    Each of the orders will be attached to the trade object for the duration of the trade - hence increasing memory usage.
+    Trades with long duration and 10s or even 100ds of position adjustments are therefore not recommended, and should be closed at regular intervals to not affect performance.
+
 ``` python
 from freqtrade.persistence import Trade
 
@@ -822,7 +839,8 @@ class DigDeeperStrategy(IStrategy):
                               min_stake: Optional[float], max_stake: float,
                               current_entry_rate: float, current_exit_rate: float,
                               current_entry_profit: float, current_exit_profit: float,
-                              **kwargs) -> Optional[float]:
+                              **kwargs
+                              ) -> Union[Optional[float], Tuple[Optional[float], Optional[str]]]:
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
@@ -848,11 +866,12 @@ class DigDeeperStrategy(IStrategy):
         :return float: Stake amount to adjust your trade,
                        Positive values to increase position, Negative values to decrease position.
                        Return None for no action.
+                       Optionally, return a tuple with a 2nd element with an order reason
         """
 
         if current_profit > 0.05 and trade.nr_of_successful_exits == 0:
             # Take half of the profit at +5%
-            return -(trade.stake_amount / 2)
+            return -(trade.stake_amount / 2), 'half_profit_5%'
 
         if current_profit > -0.05:
             return None
@@ -880,7 +899,7 @@ class DigDeeperStrategy(IStrategy):
             stake_amount = filled_entries[0].stake_amount
             # This then calculates current safety order size
             stake_amount = stake_amount * (1 + (count_of_entries * 0.25))
-            return stake_amount
+            return stake_amount, '1/3rd_increase'
         except Exception as exception:
             return None
 

docs/strategy-customization.md

@@ -156,9 +156,9 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame
 
 Out of the box, freqtrade installs the following technical libraries:
 
-* [ta-lib](http://mrjbq7.github.io/ta-lib/)
-* [pandas-ta](https://twopirllc.github.io/pandas-ta/)
-* [technical](https://github.com/freqtrade/technical/)
+- [ta-lib](https://ta-lib.github.io/ta-lib-python/)
+- [pandas-ta](https://twopirllc.github.io/pandas-ta/)
+- [technical](https://github.com/freqtrade/technical/)
 
 Additional technical libraries can be installed as necessary, or custom indicators may be written / invented by the strategy author.
 
@@ -173,7 +173,7 @@ You can use [recursive-analysis](recursive-analysis.md) to check and find the co
 In this example strategy, this should be set to 400 (`startup_candle_count = 400`), since the minimum needed history for ema100 calculation to make sure the value is correct is 400 candles.
 
 ``` python
-    dataframe['ema100'] = ta.EMA(dataframe, timeperiod=400)
+    dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
 ```
 
 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt.
@@ -367,6 +367,11 @@ class AwesomeStrategy(IStrategy):
     }
 ```
 
+??? info "Orders that don't fill immediately"
+    `minimal_roi` will take the `trade.open_date` as reference, which is the time the trade was initialized / the first order for this trade was placed.  
+    This will also hold true for limit orders that don't fill immediately  (usually in combination with "off-spot" prices through `custom_entry_price()`), as well as for cases where the initial order is replaced through `adjust_entry_price()`.
+    The time used will still be from the initial `trade.open_date` (when the initial order was first placed), not from the newly placed order date.
+
 ### Stoploss
 
 Setting a stoploss is highly recommended to protect your capital from strong moves against you.
@@ -486,17 +491,18 @@ for more information.
 
         :param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe.
         :param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use
-        current pair.
+                    current pair. Also supports limited pair format strings (see below)
         :param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not
         specified, defaults to:
-        * {base}_{quote}_{column}_{timeframe} if asset is specified. 
+        * {base}_{quote}_{column}_{timeframe} if asset is specified.
         * {column}_{timeframe} if asset is not specified.
-        Format string supports these format variables:
-        * {asset} - full name of the asset, for example 'BTC/USDT'.
+        Pair format supports these format variables:
         * {base} - base currency in lower case, for example 'eth'.
         * {BASE} - same as {base}, except in upper case.
         * {quote} - quote currency in lower case, for example 'usdt'.
         * {QUOTE} - same as {quote}, except in upper case.
+        Format string additionally supports this variables.
+        * {asset} - full name of the asset, for example 'BTC/USDT'.
         * {column} - name of dataframe column.
         * {timeframe} - timeframe of informative dataframe.
         :param ffill: ffill dataframe after merging informative pair.
@@ -1003,11 +1009,15 @@ This is a common pain-point, which can cause huge differences between backtestin
 
 The following lists some common patterns which should be avoided to prevent frustration:
 
-- don't use `shift(-1)`. This uses data from the future, which is not available.
-- don't use `.iloc[-1]` or any other absolute position in the dataframe, this will be different between dry-run and backtesting.
+- don't use `shift(-1)` or other negative values. This uses data from the future in backtesting, which is not available in dry or live modes.
+- don't use `.iloc[-1]` or any other absolute position in the dataframe within `populate_` functions, as this will be different between dry-run and backtesting. Absolute `iloc` indexing is safe to use in callbacks however - see [Strategy Callbacks](strategy-callbacks.md).
 - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling(<window>).mean()` instead
 - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead.
 
+!!! Tip "Identifying problems"
+    You may also want to check the 2 helper commands [lookahead-analysis](lookahead-analysis.md) and [recursive-analysis](recursive-analysis.md), which can each help you figure out problems with your strategy in different ways.
+    Please treat them as what they are - helpers to identify most common problems. A negative result of each does not guarantee that there's none of the above errors included.
+
 ### Colliding signals
 
 When conflicting signals collide (e.g. both `'enter_long'` and `'exit_long'` are 1), freqtrade will do nothing and ignore the entry signal. This will avoid trades that enter, and exit immediately. Obviously, this can potentially lead to missed entries.

docs/strategy_migration.md

@@ -570,7 +570,7 @@ def populate_any_indicators(
 ```
 
 1. Features - Move to `feature_engineering_expand_all`
-2. Basic features, not expanded across `include_periods_candles` - move to`feature_engineering_expand_basic()`.
+2. Basic features, not expanded across `indicator_periods_candles` - move to`feature_engineering_expand_basic()`.
 3. Standard features which should not be expanded - move to `feature_engineering_standard()`.
 4. Targets - Move this part to `set_freqai_targets()`.
 

docs/telegram-usage.md

@@ -175,6 +175,7 @@ official commands. You can ask at any moment for help with `/help`.
 | `/status` | Lists all open trades
 | `/status <trade_id>` | Lists one or more specific trade. Separate multiple <trade_id> with a blank space.
 | `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**)
+| `/order <trade_id>` | Lists orders of one or more specific trade. Separate multiple <trade_id> with a blank space.
 | `/trades [limit]` | List all recently closed trades in a table format.
 | `/count` | Displays number of trades used and available
 | `/locks` | Show currently locked pairs.

docs/utils.md

@@ -54,7 +54,7 @@ optional arguments:
 ### Create config examples
 
 ```
-$ freqtrade new-config --config config_binance.json
+$ freqtrade new-config --config user_data/config_binance.json
 
 ? Do you want to enable Dry-run (simulated trades)?  Yes
 ? Please insert your stake currency: BTC
@@ -242,7 +242,6 @@ bitkk            True     missing opt: fetchMyTrades
 bitmart          True
 bitmax           True     missing opt: fetchMyTrades
 bitpanda         True
-bittrex          True
 bitvavo          True
 bitz             True     missing opt: fetchMyTrades
 btcalpha         True     missing opt: fetchTicker, fetchTickers
@@ -324,7 +323,6 @@ bitpanda            True
 bitso               False    missing: fetchOHLCV
 bitstamp            True     missing opt: fetchTickers
 bitstamp1           False    missing: fetchOrder, fetchOHLCV
-bittrex             True
 bitvavo             True
 bitz                True     missing opt: fetchMyTrades
 bl3p                False    missing: fetchOrder, fetchOHLCV
@@ -427,25 +425,33 @@ zb                  True     missing opt: fetchMyTrades
 Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange.
 
 ```
-usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1]
+usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                 [-d PATH] [--userdir PATH]
+                                 [--exchange EXCHANGE] [-1]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   -1, --one-column      Print output in one column.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details.
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
+                        'syslog', 'journald'. See the documentation for more
+                        details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-`
-                        to read config from stdin.
-  -d PATH, --datadir PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH, --data-dir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
 
+
 ```
 
 * Example: see the timeframes for the 'binance' exchange, set in the configuration file:
@@ -479,20 +485,17 @@ usage: freqtrade list-markets [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                               [-d PATH] [--userdir PATH] [--exchange EXCHANGE]
                               [--print-list] [--print-json] [-1] [--print-csv]
                               [--base BASE_CURRENCY [BASE_CURRENCY ...]]
-                              [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a]
-                              [--trading-mode {spot,margin,futures}]
-
+                              [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]]
+                              [-a] [--trading-mode {spot,margin,futures}]
 usage: freqtrade list-pairs [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                             [-d PATH] [--userdir PATH] [--exchange EXCHANGE]
                             [--print-list] [--print-json] [-1] [--print-csv]
                             [--base BASE_CURRENCY [BASE_CURRENCY ...]]
                             [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a]
                             [--trading-mode {spot,margin,futures}]
-
-optional arguments:
+options:
   -h, --help            show this help message and exit
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   --print-list          Print list of pairs or market symbols. By default data
                         is printed in the tabular format.
   --print-json          Print list of pairs or market symbols in JSON format.
@@ -504,20 +507,22 @@ optional arguments:
                         Specify quote currency(-ies). Space-separated list.
   -a, --all             Print all pairs or market symbols. By default only
                         active ones are shown.
-  --trading-mode {spot,margin,futures}
+  --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
                         Select Trading mode
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
   -c PATH, --config PATH
-                        Specify configuration file (default: `config.json`).
-                        Multiple --config options may be used. Can be set to
-                        `-` to read config from stdin.
-  -d PATH, --datadir PATH
+                        Specify configuration file (default:
+                        `userdir/config.json` or `config.json` whichever
+                        exists). Multiple --config options may be used. Can be
+                        set to `-` to read config from stdin.
+  -d PATH, --datadir PATH, --data-dir PATH
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
@@ -532,7 +537,7 @@ Pairs/markets are sorted by its symbol string in the printed output.
 ### Examples
 
 * Print the list of active pairs with quote currency USD on exchange, specified in the default
-configuration file (i.e. pairs on the "Bittrex" exchange) in JSON format:
+configuration file (i.e. pairs on the "Binance" exchange) in JSON format:
 
 ```
 $ freqtrade list-pairs --quote USD --print-json
@@ -564,7 +569,7 @@ usage: freqtrade test-pairlist [-h] [--userdir PATH] [-v] [-c PATH]
                                [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]]
                                [-1] [--print-json] [--exchange EXCHANGE]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
@@ -578,8 +583,7 @@ optional arguments:
                         Specify quote currency(-ies). Space-separated list.
   -1, --one-column      Print output in one column.
   --print-json          Print list of pairs or market symbols in JSON format.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
 
 ```
 
@@ -986,11 +990,7 @@ options:
   -h, --help            show this help message and exit
   --strategy-list STRATEGY_LIST [STRATEGY_LIST ...]
                         Provide a space-separated list of strategies to
-                        backtest. Please note that timeframe needs to be set
-                        either in config or via command line. When using this
-                        together with `--export trades`, the strategy-name is
-                        injected into the filename (so `backtest-data.json`
-                        becomes `backtest-data-SampleStrategy.json`
+                        be converted.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/webhook-config.md

@@ -134,6 +134,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -155,6 +156,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -176,6 +178,7 @@ Possible parameters are:
 * `stake_amount`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `order_type`
 * `current_rate`
@@ -199,6 +202,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -224,6 +228,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -249,6 +254,7 @@ Possible parameters are:
 * `profit_ratio`
 * `stake_currency`
 * `base_currency`
+* `quote_currency`
 * `fiat_currency`
 * `exit_reason`
 * `order_type`
@@ -302,6 +308,7 @@ You can configure this as follows:
 ```
 
 The above represents the default (`exit_fill` and `entry_fill` are optional and will default to the above configuration) - modifications are obviously possible.
+To disable either of the two default values (`entry_fill` / `exit_fill`), you can assign them an empty array (`exit_fill: []`).
 
 Available fields correspond to the fields for webhooks and are documented in the corresponding webhook sections.
 

docs/windows_installation.md

@@ -22,7 +22,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 ### 2. Install ta-lib
 
-Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
+Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 
 As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.9'
+__version__ = '2024.2'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/commands/arguments.py

@@ -65,8 +65,8 @@ ARGS_BUILD_CONFIG = ["config"]
 
 ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
 
+ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase", "exchange"]
 ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
-
 ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
 
 ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
@@ -219,27 +219,35 @@ class Arguments:
                                                 )
 
         # Add trade subcommand
-        trade_cmd = subparsers.add_parser('trade', help='Trade module.',
-                                          parents=[_common_parser, _strategy_parser])
+        trade_cmd = subparsers.add_parser(
+            'trade',
+            help='Trade module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         trade_cmd.set_defaults(func=start_trading)
         self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd)
 
         # add create-userdir subcommand
-        create_userdir_cmd = subparsers.add_parser('create-userdir',
-                                                   help="Create user-data directory.",
-                                                   )
+        create_userdir_cmd = subparsers.add_parser(
+            'create-userdir',
+            help="Create user-data directory.",
+        )
         create_userdir_cmd.set_defaults(func=start_create_userdir)
         self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd)
 
         # add new-config subcommand
-        build_config_cmd = subparsers.add_parser('new-config',
-                                                 help="Create new config")
+        build_config_cmd = subparsers.add_parser(
+            'new-config',
+            help="Create new config",
+        )
         build_config_cmd.set_defaults(func=start_new_config)
         self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
 
         # add new-strategy subcommand
-        build_strategy_cmd = subparsers.add_parser('new-strategy',
-                                                   help="Create new strategy")
+        build_strategy_cmd = subparsers.add_parser(
+            'new-strategy',
+            help="Create new strategy",
+        )
         build_strategy_cmd.set_defaults(func=start_new_strategy)
         self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd)
 
@@ -268,7 +276,7 @@ class Arguments:
             parents=[_common_parser],
         )
         convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
-        self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
+        self._build_args(optionlist=ARGS_CONVERT_DATA_TRADES, parser=convert_trade_data_cmd)
 
         # Add trades-to-ohlcv subcommand
         convert_trade_data_cmd = subparsers.add_parser(
@@ -289,8 +297,11 @@ class Arguments:
         self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
 
         # Add backtesting subcommand
-        backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
-                                                parents=[_common_parser, _strategy_parser])
+        backtesting_cmd = subparsers.add_parser(
+            'backtesting',
+            help='Backtesting module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         backtesting_cmd.set_defaults(func=start_backtesting)
         self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
 
@@ -304,22 +315,29 @@ class Arguments:
         self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
 
         # Add backtesting analysis subcommand
-        analysis_cmd = subparsers.add_parser('backtesting-analysis',
-                                             help='Backtest Analysis module.',
-                                             parents=[_common_parser])
+        analysis_cmd = subparsers.add_parser(
+            'backtesting-analysis',
+            help='Backtest Analysis module.',
+            parents=[_common_parser]
+        )
         analysis_cmd.set_defaults(func=start_analysis_entries_exits)
         self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
 
         # Add edge subcommand
-        edge_cmd = subparsers.add_parser('edge', help='Edge module.',
-                                         parents=[_common_parser, _strategy_parser])
+        edge_cmd = subparsers.add_parser(
+            'edge',
+            help='Edge module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         edge_cmd.set_defaults(func=start_edge)
         self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd)
 
         # Add hyperopt subcommand
-        hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.',
-                                             parents=[_common_parser, _strategy_parser],
-                                             )
+        hyperopt_cmd = subparsers.add_parser(
+            'hyperopt',
+            help='Hyperopt module.',
+            parents=[_common_parser, _strategy_parser],
+        )
         hyperopt_cmd.set_defaults(func=start_hyperopt)
         self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd)
 
@@ -447,16 +465,20 @@ class Arguments:
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
 
         # Add webserver subcommand
-        webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
-                                              parents=[_common_parser])
+        webserver_cmd = subparsers.add_parser(
+            'webserver',
+            help='Webserver module.',
+            parents=[_common_parser]
+        )
         webserver_cmd.set_defaults(func=start_webserver)
         self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
 
         # Add strategy_updater subcommand
-        strategy_updater_cmd = subparsers.add_parser('strategy-updater',
-                                                     help='updates outdated strategy'
-                                                          'files to the current version',
-                                                     parents=[_common_parser])
+        strategy_updater_cmd = subparsers.add_parser(
+            'strategy-updater',
+            help='updates outdated strategy files to the current version',
+            parents=[_common_parser]
+        )
         strategy_updater_cmd.set_defaults(func=start_strategy_update)
         self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd)
 
@@ -464,8 +486,8 @@ class Arguments:
         lookahead_analayis_cmd = subparsers.add_parser(
             'lookahead-analysis',
             help="Check for potential look ahead bias.",
-            parents=[_common_parser, _strategy_parser])
-
+            parents=[_common_parser, _strategy_parser]
+        )
         lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis)
 
         self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
@@ -475,8 +497,8 @@ class Arguments:
         recursive_analayis_cmd = subparsers.add_parser(
             'recursive-analysis',
             help="Check for potential recursive formula issue.",
-            parents=[_common_parser, _strategy_parser])
-
+            parents=[_common_parser, _strategy_parser]
+        )
         recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
 
         self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,

freqtrade/commands/build_config_commands.py

@@ -108,9 +108,8 @@ def ask_user_config() -> Dict[str, Any]:
             "choices": [
                 "binance",
                 "binanceus",
-                "bittrex",
                 "gate",
-                "huobi",
+                "htx",
                 "kraken",
                 "kucoin",
                 "okx",

freqtrade/commands/cli_options.py

@@ -421,6 +421,12 @@ AVAILABLE_CLI_OPTIONS = {
              'desired timeframe as specified as --timeframes/-t.',
         action='store_true',
     ),
+    "format_from_trades": Arg(
+        '--format-from',
+        help='Source format for data conversion.',
+        choices=constants.AVAILABLE_DATAHANDLERS + ['kraken_csv'],
+        required=True,
+    ),
     "format_from": Arg(
         '--format-from',
         help='Source format for data conversion.',

freqtrade/commands/data_commands.py

@@ -12,7 +12,7 @@ from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.resolvers import ExchangeResolver
-from freqtrade.util.binance_mig import migrate_binance_futures_data
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -78,14 +78,14 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
-        migrate_binance_futures_data(config)
+        migrate_data(config)
         convert_ohlcv_format(config,
                              convert_from=args['format_from'],
                              convert_to=args['format_to'],
                              erase=args['erase'])
     else:
         convert_trades_format(config,
-                              convert_from=args['format_from'], convert_to=args['format_to'],
+                              convert_from=args['format_from_trades'], convert_to=args['format_to'],
                               erase=args['erase'])
 
 
@@ -134,10 +134,10 @@ def start_list_data(args: Dict[str, Any]) -> None:
         print(tabulate([
             (pair, timeframe, candle_type,
                 start.strftime(DATETIME_PRINT_FORMAT),
-                end.strftime(DATETIME_PRINT_FORMAT))
-            for pair, timeframe, candle_type, start, end in sorted(
+                end.strftime(DATETIME_PRINT_FORMAT), length)
+            for pair, timeframe, candle_type, start, end, length in sorted(
                 paircombs1,
                 key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
             ],
-            headers=("Pair", "Timeframe", "Type", 'From', 'To'),
+            headers=("Pair", "Timeframe", "Type", 'From', 'To', 'Candles'),
             tablefmt='psql', stralign='right'))

freqtrade/commands/optimize_commands.py

@@ -5,7 +5,7 @@ from freqtrade import constants
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import round_coin_value
+from freqtrade.util import fmt_coin
 
 
 logger = logging.getLogger(__name__)
@@ -29,8 +29,8 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
         # tradable_balance_ratio
         if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT
                 and config['stake_amount'] > wallet_size):
-            wallet = round_coin_value(wallet_size, config['stake_currency'])
-            stake = round_coin_value(config['stake_amount'], config['stake_currency'])
+            wallet = fmt_coin(wallet_size, config['stake_currency'])
+            stake = fmt_coin(config['stake_amount'], config['stake_currency'])
             raise OperationalException(
                 f"Starting balance ({wallet}) is smaller than stake_amount {stake}. "
                 f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`."
@@ -140,7 +140,7 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    from freqtrade.optimize.lookahead_analysis_helpers import LookaheadAnalysisSubFunctions
+    from freqtrade.optimize.analysis.lookahead_helpers import LookaheadAnalysisSubFunctions
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     LookaheadAnalysisSubFunctions.start(config)
@@ -152,7 +152,7 @@ def start_recursive_analysis(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions
+    from freqtrade.optimize.analysis.recursive_helpers import RecursiveAnalysisSubFunctions
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     RecursiveAnalysisSubFunctions.start(config)

freqtrade/commands/pairlist_commands.py

@@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     Test Pairlist configuration
     """
+    from freqtrade.persistence import FtNoDBContext
     from freqtrade.plugins.pairlistmanager import PairListManager
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
@@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     if not quote_currencies:
         quote_currencies = [config.get('stake_currency')]
     results = {}
-    for curr in quote_currencies:
-        config['stake_currency'] = curr
-        pairlists = PairListManager(exchange, config)
-        pairlists.refresh_pairlist()
-        results[curr] = pairlists.whitelist
+    with FtNoDBContext():
+        for curr in quote_currencies:
+            config['stake_currency'] = curr
+            pairlists = PairListManager(exchange, config)
+            pairlists.refresh_pairlist()
+            results[curr] = pairlists.whitelist
 
     for curr, pairlist in results.items():
         if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):

freqtrade/configuration/config_validation.py

@@ -67,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
         )
 
 
-def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None:
+def validate_config_consistency(conf: Dict[str, Any], *, preliminary: bool = False) -> None:
     """
     Validate the configuration consistency.
     Should be ran after loading both configuration and strategy,
@@ -86,7 +86,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
     _validate_ask_orderbook(conf)
     _validate_freqai_hyperopt(conf)
     _validate_freqai_backtest(conf)
-    _validate_freqai_include_timeframes(conf)
+    _validate_freqai_include_timeframes(conf, preliminary=preliminary)
     _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
 
@@ -335,7 +335,7 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
             'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
 
 
-def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
+def _validate_freqai_include_timeframes(conf: Dict[str, Any], preliminary: bool) -> None:
     freqai_enabled = conf.get('freqai', {}).get('enabled', False)
     if freqai_enabled:
         main_tf = conf.get('timeframe', '5m')
@@ -355,7 +355,7 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
                 f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}")
 
         # Ensure that the base timeframe is included in the include_timeframes list
-        if main_tf not in freqai_include_timeframes:
+        if not preliminary and main_tf not in freqai_include_timeframes:
             feature_parameters = conf.get('freqai', {}).get('feature_parameters', {})
             include_timeframes = [main_tf] + freqai_include_timeframes
             conf.get('freqai', {}).get('feature_parameters', {}) \

freqtrade/configuration/configuration.py

@@ -5,7 +5,7 @@ import logging
 import warnings
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from freqtrade import constants
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
@@ -68,8 +68,10 @@ class Configuration:
         config: Config = load_from_files(self.args.get("config", []))
 
         # Load environment variables
-        env_data = enironment_vars_to_dict()
-        config = deep_merge_dicts(env_data, config)
+        from freqtrade.commands.arguments import NO_CONF_ALLOWED
+        if self.args.get('command') not in NO_CONF_ALLOWED:
+            env_data = enironment_vars_to_dict()
+            config = deep_merge_dicts(env_data, config)
 
         # Normalize config
         if 'internals' not in config:
@@ -233,54 +235,37 @@ class Configuration:
             except ValueError:
                 pass
 
-        self._args_to_config(config, argname='timeframe_detail',
-                             logstring='Parameter --timeframe-detail detected, '
-                             'using {} for intra-candle backtesting ...')
+        configurations = [
+            ('timeframe_detail',
+             'Parameter --timeframe-detail detected, using {} for intra-candle backtesting ...'),
+            ('backtest_show_pair_list', 'Parameter --show-pair-list detected.'),
+            ('stake_amount',
+             'Parameter --stake-amount detected, overriding stake_amount to: {} ...'),
+            ('dry_run_wallet',
+             'Parameter --dry-run-wallet detected, overriding dry_run_wallet to: {} ...'),
+            ('fee', 'Parameter --fee detected, setting fee to: {} ...'),
+            ('timerange', 'Parameter --timerange detected: {} ...'),
+            ]
 
-        self._args_to_config(config, argname='backtest_show_pair_list',
-                             logstring='Parameter --show-pair-list detected.')
-
-        self._args_to_config(config, argname='stake_amount',
-                             logstring='Parameter --stake-amount detected, '
-                             'overriding stake_amount to: {} ...')
-        self._args_to_config(config, argname='dry_run_wallet',
-                             logstring='Parameter --dry-run-wallet detected, '
-                             'overriding dry_run_wallet to: {} ...')
-        self._args_to_config(config, argname='fee',
-                             logstring='Parameter --fee detected, '
-                             'setting fee to: {} ...')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Parameter --timerange detected: {} ...')
+        self._args_to_config_loop(config, configurations)
 
         self._process_datadir_options(config)
 
         self._args_to_config(config, argname='strategy_list',
                              logstring='Using strategy list of {} strategies', logfun=len)
 
-        self._args_to_config(
-            config,
-            argname='recursive_strategy_search',
-            logstring='Recursively searching for a strategy in the strategies folder.',
-        )
-
-        self._args_to_config(config, argname='timeframe',
-                             logstring='Overriding timeframe with Command line argument')
-
-        self._args_to_config(config, argname='export',
-                             logstring='Parameter --export detected: {} ...')
-
-        self._args_to_config(config, argname='backtest_breakdown',
-                             logstring='Parameter --breakdown detected ...')
-
-        self._args_to_config(config, argname='backtest_cache',
-                             logstring='Parameter --cache={} detected ...')
-
-        self._args_to_config(config, argname='disableparamexport',
-                             logstring='Parameter --disableparamexport detected: {} ...')
-
-        self._args_to_config(config, argname='freqai_backtest_live_models',
-                             logstring='Parameter --freqai-backtest-live-models detected ...')
+        configurations = [
+            ('recursive_strategy_search',
+             'Recursively searching for a strategy in the strategies folder.'),
+            ('timeframe', 'Overriding timeframe with Command line argument'),
+            ('export', 'Parameter --export detected: {} ...'),
+            ('backtest_breakdown', 'Parameter --breakdown detected ...'),
+            ('backtest_cache', 'Parameter --cache={} detected ...'),
+            ('disableparamexport', 'Parameter --disableparamexport detected: {} ...'),
+            ('freqai_backtest_live_models',
+             'Parameter --freqai-backtest-live-models detected ...'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
         # Edge section:
         if 'stoploss_range' in self.args and self.args["stoploss_range"]:
@@ -291,31 +276,18 @@ class Configuration:
             logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"])
 
         # Hyperopt section
-        self._args_to_config(config, argname='hyperopt',
-                             logstring='Using Hyperopt class name: {}')
 
-        self._args_to_config(config, argname='hyperopt_path',
-                             logstring='Using additional Hyperopt lookup path: {}')
-
-        self._args_to_config(config, argname='hyperoptexportfilename',
-                             logstring='Using hyperopt file: {}')
-
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
-                             logstring='Saving lookahead analysis results into {} ...')
-
-        self._args_to_config(config, argname='epochs',
-                             logstring='Parameter --epochs detected ... '
-                             'Will run Hyperopt with for {} epochs ...'
-                             )
-
-        self._args_to_config(config, argname='spaces',
-                             logstring='Parameter -s/--spaces detected: {}')
-
-        self._args_to_config(config, argname='analyze_per_epoch',
-                             logstring='Parameter --analyze-per-epoch detected.')
-
-        self._args_to_config(config, argname='print_all',
-                             logstring='Parameter --print-all detected ...')
+        configurations = [
+            ('hyperopt', 'Using Hyperopt class name: {}'),
+            ('hyperopt_path', 'Using additional Hyperopt lookup path: {}'),
+            ('hyperoptexportfilename', 'Using hyperopt file: {}'),
+            ('lookahead_analysis_exportfilename', 'Saving lookahead analysis results into {} ...'),
+            ('epochs', 'Parameter --epochs detected ... Will run Hyperopt with for {} epochs ...'),
+            ('spaces', 'Parameter -s/--spaces detected: {}'),
+            ('analyze_per_epoch', 'Parameter --analyze-per-epoch detected.'),
+            ('print_all', 'Parameter --print-all detected ...'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
         if 'print_colorized' in self.args and not self.args["print_colorized"]:
             logger.info('Parameter --no-color detected ...')
@@ -323,123 +295,55 @@ class Configuration:
         else:
             config.update({'print_colorized': True})
 
-        self._args_to_config(config, argname='print_json',
-                             logstring='Parameter --print-json detected ...')
+        configurations = [
+            ('print_json', 'Parameter --print-json detected ...'),
+            ('export_csv', 'Parameter --export-csv detected: {}'),
+            ('hyperopt_jobs', 'Parameter -j/--job-workers detected: {}'),
+            ('hyperopt_random_state', 'Parameter --random-state detected: {}'),
+            ('hyperopt_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_loss', 'Using Hyperopt loss class name: {}'),
+            ('hyperopt_show_index', 'Parameter -n/--index detected: {}'),
+            ('hyperopt_list_best', 'Parameter --best detected: {}'),
+            ('hyperopt_list_profitable', 'Parameter --profitable detected: {}'),
+            ('hyperopt_list_min_trades', 'Parameter --min-trades detected: {}'),
+            ('hyperopt_list_max_trades', 'Parameter --max-trades detected: {}'),
+            ('hyperopt_list_min_avg_time', 'Parameter --min-avg-time detected: {}'),
+            ('hyperopt_list_max_avg_time', 'Parameter --max-avg-time detected: {}'),
+            ('hyperopt_list_min_avg_profit', 'Parameter --min-avg-profit detected: {}'),
+            ('hyperopt_list_max_avg_profit', 'Parameter --max-avg-profit detected: {}'),
+            ('hyperopt_list_min_total_profit', 'Parameter --min-total-profit detected: {}'),
+            ('hyperopt_list_max_total_profit', 'Parameter --max-total-profit detected: {}'),
+            ('hyperopt_list_min_objective', 'Parameter --min-objective detected: {}'),
+            ('hyperopt_list_max_objective', 'Parameter --max-objective detected: {}'),
+            ('hyperopt_list_no_details', 'Parameter --no-details detected: {}'),
+            ('hyperopt_show_no_header', 'Parameter --no-header detected: {}'),
+            ('hyperopt_ignore_missing_space', 'Paramter --ignore-missing-space detected: {}'),
+        ]
 
-        self._args_to_config(config, argname='export_csv',
-                             logstring='Parameter --export-csv detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_jobs',
-                             logstring='Parameter -j/--job-workers detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_random_state',
-                             logstring='Parameter --random-state detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_loss',
-                             logstring='Using Hyperopt loss class name: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_index',
-                             logstring='Parameter -n/--index detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_best',
-                             logstring='Parameter --best detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_profitable',
-                             logstring='Parameter --profitable detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_trades',
-                             logstring='Parameter --min-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_trades',
-                             logstring='Parameter --max-trades detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_time',
-                             logstring='Parameter --min-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_time',
-                             logstring='Parameter --max-avg-time detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_avg_profit',
-                             logstring='Parameter --min-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_avg_profit',
-                             logstring='Parameter --max-avg-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_total_profit',
-                             logstring='Parameter --min-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_total_profit',
-                             logstring='Parameter --max-total-profit detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_min_objective',
-                             logstring='Parameter --min-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_max_objective',
-                             logstring='Parameter --max-objective detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_list_no_details',
-                             logstring='Parameter --no-details detected: {}')
-
-        self._args_to_config(config, argname='hyperopt_show_no_header',
-                             logstring='Parameter --no-header detected: {}')
-
-        self._args_to_config(config, argname="hyperopt_ignore_missing_space",
-                             logstring="Paramter --ignore-missing-space detected: {}")
+        self._args_to_config_loop(config, configurations)
 
     def _process_plot_options(self, config: Config) -> None:
 
-        self._args_to_config(config, argname='pairs',
-                             logstring='Using pairs {}')
-
-        self._args_to_config(config, argname='indicators1',
-                             logstring='Using indicators1: {}')
-
-        self._args_to_config(config, argname='indicators2',
-                             logstring='Using indicators2: {}')
-
-        self._args_to_config(config, argname='trade_ids',
-                             logstring='Filtering on trade_ids: {}')
-
-        self._args_to_config(config, argname='plot_limit',
-                             logstring='Limiting plot to: {}')
-
-        self._args_to_config(config, argname='plot_auto_open',
-                             logstring='Parameter --auto-open detected.')
-
-        self._args_to_config(config, argname='trade_source',
-                             logstring='Using trades from: {}')
-
-        self._args_to_config(config, argname='prepend_data',
-                             logstring='Prepend detected. Allowing data prepending.')
-        self._args_to_config(config, argname='erase',
-                             logstring='Erase detected. Deleting existing data.')
-
-        self._args_to_config(config, argname='no_trades',
-                             logstring='Parameter --no-trades detected.')
-
-        self._args_to_config(config, argname='timeframes',
-                             logstring='timeframes --timeframes: {}')
-
-        self._args_to_config(config, argname='days',
-                             logstring='Detected --days: {}')
-
-        self._args_to_config(config, argname='include_inactive',
-                             logstring='Detected --include-inactive-pairs: {}')
-
-        self._args_to_config(config, argname='download_trades',
-                             logstring='Detected --dl-trades: {}')
-
-        self._args_to_config(config, argname='dataformat_ohlcv',
-                             logstring='Using "{}" to store OHLCV data.')
-
-        self._args_to_config(config, argname='dataformat_trades',
-                             logstring='Using "{}" to store trades data.')
-
-        self._args_to_config(config, argname='show_timerange',
-                             logstring='Detected --show-timerange')
+        configurations = [
+            ('pairs', 'Using pairs {}'),
+            ('indicators1', 'Using indicators1: {}'),
+            ('indicators2', 'Using indicators2: {}'),
+            ('trade_ids', 'Filtering on trade_ids: {}'),
+            ('plot_limit', 'Limiting plot to: {}'),
+            ('plot_auto_open', 'Parameter --auto-open detected.'),
+            ('trade_source', 'Using trades from: {}'),
+            ('prepend_data', 'Prepend detected. Allowing data prepending.'),
+            ('erase', 'Erase detected. Deleting existing data.'),
+            ('no_trades', 'Parameter --no-trades detected.'),
+            ('timeframes', 'timeframes --timeframes: {}'),
+            ('days', 'Detected --days: {}'),
+            ('include_inactive', 'Detected --include-inactive-pairs: {}'),
+            ('download_trades', 'Detected --dl-trades: {}'),
+            ('dataformat_ohlcv', 'Using "{}" to store OHLCV data.'),
+            ('dataformat_trades', 'Using "{}" to store trades data.'),
+            ('show_timerange', 'Detected --show-timerange'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
     def _process_data_options(self, config: Config) -> None:
         self._args_to_config(config, argname='new_pairs_days',
@@ -453,45 +357,27 @@ class Configuration:
                              logstring='Detected --candle-types: {}')
 
     def _process_analyze_options(self, config: Config) -> None:
-        self._args_to_config(config, argname='analysis_groups',
-                             logstring='Analysis reason groups: {}')
+        configurations = [
+            ('analysis_groups', 'Analysis reason groups: {}'),
+            ('enter_reason_list', 'Analysis enter tag list: {}'),
+            ('exit_reason_list', 'Analysis exit tag list: {}'),
+            ('indicator_list', 'Analysis indicator list: {}'),
+            ('timerange', 'Filter trades by timerange: {}'),
+            ('analysis_rejected', 'Analyse rejected signals: {}'),
+            ('analysis_to_csv', 'Store analysis tables to CSV: {}'),
+            ('analysis_csv_path', 'Path to store analysis CSVs: {}'),
+            # Lookahead analysis results
+            ('targeted_trade_amount', 'Targeted Trade amount: {}'),
+            ('minimum_trade_amount', 'Minimum Trade amount: {}'),
+            ('lookahead_analysis_exportfilename', 'Path to store lookahead-analysis-results: {}'),
+            ('startup_candle', 'Startup candle to be used on recursive analysis: {}'),
+        ]
+        self._args_to_config_loop(config, configurations)
 
-        self._args_to_config(config, argname='enter_reason_list',
-                             logstring='Analysis enter tag list: {}')
+    def _args_to_config_loop(self, config, configurations: List[Tuple[str, str]]) -> None:
 
-        self._args_to_config(config, argname='exit_reason_list',
-                             logstring='Analysis exit tag list: {}')
-
-        self._args_to_config(config, argname='indicator_list',
-                             logstring='Analysis indicator list: {}')
-
-        self._args_to_config(config, argname='timerange',
-                             logstring='Filter trades by timerange: {}')
-
-        self._args_to_config(config, argname='analysis_rejected',
-                             logstring='Analyse rejected signals: {}')
-
-        self._args_to_config(config, argname='analysis_to_csv',
-                             logstring='Store analysis tables to CSV: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
-        self._args_to_config(config, argname='analysis_csv_path',
-                             logstring='Path to store analysis CSVs: {}')
-
-        # Lookahead analysis results
-        self._args_to_config(config, argname='targeted_trade_amount',
-                             logstring='Targeted Trade amount: {}')
-
-        self._args_to_config(config, argname='minimum_trade_amount',
-                             logstring='Minimum Trade amount: {}')
-
-        self._args_to_config(config, argname='lookahead_analysis_exportfilename',
-                             logstring='Path to store lookahead-analysis-results: {}')
-
-        self._args_to_config(config, argname='startup_candle',
-                             logstring='Startup candle to be used on recursive analysis: {}')
+        for argname, logstring in configurations:
+            self._args_to_config(config, argname=argname, logstring=logstring)
 
     def _process_runmode(self, config: Config) -> None:
 

freqtrade/configuration/environment_vars.py

@@ -9,7 +9,7 @@ from freqtrade.misc import deep_merge_dicts
 logger = logging.getLogger(__name__)
 
 
-def get_var_typed(val):
+def _get_var_typed(val):
     try:
         return int(val)
     except ValueError:
@@ -24,7 +24,7 @@ def get_var_typed(val):
     return val
 
 
-def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
     """
     Environment variables must be prefixed with FREQTRADE.
     FREQTRADE__{section}__{key}
@@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
             logger.info(f"Loading variable '{env_var}'")
             key = env_var.replace(prefix, '')
             for k in reversed(key.split('__')):
-                val = {k.lower(): get_var_typed(val)
+                val = {k.lower(): _get_var_typed(val)
                        if not isinstance(val, dict) and k not in no_convert else val}
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
@@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]:
     Relevant variables must follow the FREQTRADE__{section}__{key} pattern
     :return: Nested dict based on available and relevant variables.
     """
-    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
+    return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

freqtrade/constants.py

@@ -33,9 +33,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter',
-                       'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
-                       'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
+                       'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter',
+                       'PerformanceFilter', 'PrecisionFilter', 'PriceFilter',
+                       'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter',
+                       'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',
                          'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']
@@ -105,7 +106,7 @@ SUPPORTED_FIAT = [
     "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
     "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
     "RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR",
-    "USD", "BTC", "ETH", "XRP", "LTC", "BCH"
+    "USD", "BTC", "ETH", "XRP", "LTC", "BCH", "BNB"
 ]
 
 MINIMAL_CONFIG = {

freqtrade/data/btanalysis.py

@@ -175,36 +175,40 @@ def _get_backtest_files(dirname: Path) -> List[Path]:
     return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
 
 
-def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
-    """
-    Get backtest result read from metadata file
-    """
+def _extract_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+    metadata = load_backtest_metadata(filename)
     return [
         {
             'filename': filename.stem,
             'strategy': s,
-            'notes': v.get('notes', ''),
             'run_id': v['run_id'],
+            'notes': v.get('notes', ''),
+            # Backtest "run" time
             'backtest_start_time': v['backtest_start_time'],
-        } for s, v in load_backtest_metadata(filename).items()
+            # Backtest timerange
+            'backtest_start_ts': v.get('backtest_start_ts', None),
+            'backtest_end_ts': v.get('backtest_end_ts', None),
+            'timeframe': v.get('timeframe', None),
+            'timeframe_detail': v.get('timeframe_detail', None),
+        } for s, v in metadata.items()
     ]
 
 
+def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+    """
+    Get backtest result read from metadata file
+    """
+    return _extract_backtest_result(filename)
+
+
 def get_backtest_resultlist(dirname: Path) -> List[BacktestHistoryEntryType]:
     """
     Get list of backtest results read from metadata files
     """
     return [
-        {
-            'filename': filename.stem,
-            'strategy': s,
-            'run_id': v['run_id'],
-            'notes': v.get('notes', ''),
-            'backtest_start_time': v['backtest_start_time'],
-        }
+        result
         for filename in _get_backtest_files(dirname)
-        for s, v in load_backtest_metadata(filename).items()
-        if v
+        for result in _extract_backtest_result(filename)
     ]
 
 
@@ -326,7 +330,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                  "Please specify a strategy.")
 
         if strategy not in data['strategy']:
-            raise ValueError(f"Strategy {strategy} not available in the backtest result.")
+            raise ValueError(
+                f"Strategy {strategy} not available in the backtest result. "
+                f"Available strategies are '{','.join(data['strategy'].keys())}'"
+                )
 
         data = data['strategy'][strategy]['trades']
         df = pd.DataFrame(data)
@@ -350,10 +357,10 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
     :param timeframe: Timeframe used for backtest
     :return: dataframe with open-counts per time-period in timeframe
     """
-    from freqtrade.exchange import timeframe_to_minutes
-    timeframe_min = timeframe_to_minutes(timeframe)
+    from freqtrade.exchange import timeframe_to_resample_freq
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'],
-                                     freq=f"{timeframe_min}min"))
+                                     freq=timeframe_freq))
              for row in results[['open_date', 'close_date']].iterrows()]
     deltas = [len(x) for x in dates]
     dates = pd.Series(pd.concat(dates).values, name='date')
@@ -361,7 +368,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
 
     df2 = pd.concat([dates, df2], axis=1)
     df2 = df2.set_index('date')
-    df_final = df2.resample(f"{timeframe_min}min")[['pair']].count()
+    df_final = df2.resample(timeframe_freq)[['pair']].count()
     df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
     return df_final
 

freqtrade/data/converter/converter.py

@@ -84,7 +84,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     using the previous close as price for "open", "high" "low" and "close", volume is set to 0
 
     """
-    from freqtrade.exchange import timeframe_to_minutes
+    from freqtrade.exchange import timeframe_to_resample_freq
 
     ohlcv_dict = {
         'open': 'first',
@@ -93,13 +93,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
         'close': 'last',
         'volume': 'sum'
     }
-    timeframe_minutes = timeframe_to_minutes(timeframe)
-    resample_interval = f'{timeframe_minutes}min'
-    if timeframe_minutes >= 43200 and timeframe_minutes < 525600:
-        # Monthly candles need special treatment to stick to the 1st of the month
-        resample_interval = f'{timeframe}S'
-    elif timeframe_minutes > 43200:
-        resample_interval = timeframe
+    resample_interval = timeframe_to_resample_freq(timeframe)
     # Resample to create "NAN" values
     df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict)
 
@@ -116,8 +110,8 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     len_after = len(df)
     pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
     if len_before != len_after:
-        message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
-                   f" - {pct_missing:.2%}")
+        message = (f"Missing data fillup for {pair}, {timeframe}: "
+                   f"before: {len_before} - after: {len_after} - {pct_missing:.2%}")
         if pct_missing > 0.01:
             logger.info(message)
         else:

freqtrade/data/converter/trade_converter.py

@@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
                                  Config, TradeList)
 from freqtrade.enums import CandleType
+from freqtrade.exceptions import OperationalException
 
 
 logger = logging.getLogger(__name__)
@@ -69,14 +70,13 @@ def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame:
     :return: OHLCV Dataframe.
     :raises: ValueError if no trades are provided
     """
-    from freqtrade.exchange import timeframe_to_minutes
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    from freqtrade.exchange import timeframe_to_resample_freq
     if trades.empty:
         raise ValueError('Trade-list empty.')
     df = trades.set_index('date', drop=True)
-
-    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
-    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
+    resample_interval = timeframe_to_resample_freq(timeframe)
+    df_new = df['price'].resample(resample_interval).ohlc()
+    df_new['volume'] = df['amount'].resample(resample_interval).sum()
     df_new['date'] = df_new.index
     # Drop 0 volume rows
     df_new = df_new.dropna()
@@ -127,6 +127,16 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
     :param convert_to: Target format
     :param erase: Erase source data (does not apply if source and target format are identical)
     """
+    if convert_from == 'kraken_csv':
+        if config['exchange']['name'] != 'kraken':
+            raise OperationalException(
+                'Converting from csv is only supported for kraken.'
+                'Please refer to the documentation for details about this special mode.'
+            )
+        from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv
+        import_kraken_trades_from_csv(config, convert_to)
+        return
+
     from freqtrade.data.history.idatahandler import get_datahandler
     src = get_datahandler(config['datadir'], convert_from)
     trg = get_datahandler(config['datadir'], convert_to)

freqtrade/data/converter/trade_converter_kraken.py

@@ -0,0 +1,82 @@
+import logging
+from pathlib import Path
+
+import pandas as pd
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, Config
+from freqtrade.data.converter.trade_converter import (trades_convert_types,
+                                                      trades_df_remove_duplicates)
+from freqtrade.data.history.idatahandler import get_datahandler
+from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
+from freqtrade.resolvers import ExchangeResolver
+
+
+logger = logging.getLogger(__name__)
+
+KRAKEN_CSV_TRADE_COLUMNS = ['timestamp', 'price', 'amount']
+
+
+def import_kraken_trades_from_csv(config: Config, convert_to: str):
+    """
+    Import kraken trades from csv
+    """
+    if config['exchange']['name'] != 'kraken':
+        raise OperationalException('This function is only for the kraken exchange.')
+
+    datadir: Path = config['datadir']
+    data_handler = get_datahandler(datadir, data_format=convert_to)
+
+    tradesdir: Path = config['datadir'] / 'trades_csv'
+    exchange = ExchangeResolver.load_exchange(config, validate=False)
+    # iterate through directories in this directory
+    data_symbols = {p.stem for p in tradesdir.rglob('*.csv')}
+
+    # create pair/filename mapping
+    markets = {
+        (m['symbol'], m['altname']) for m in exchange.markets.values()
+        if m.get('altname') in data_symbols
+    }
+    logger.info(f"Found csv files for {', '.join(data_symbols)}.")
+
+    if pairs_raw := config.get('pairs'):
+        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
+        markets = {m for m in markets if m[0] in pairs}
+        if not markets:
+            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
+            return
+    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")
+
+    for pair, name in markets:
+        logger.debug(f"Converting pair {pair}, files */{name}.csv")
+        dfs = []
+        # Load and combine all csv files for this pair
+        for f in tradesdir.rglob(f"{name}.csv"):
+            df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)
+            if not df.empty:
+                dfs.append(df)
+
+        # Load existing trades data
+        if not dfs:
+            # edgecase, can only happen if the file was deleted between the above glob and here
+            logger.info(f"No data found for pair {pair}")
+            continue
+
+        trades = pd.concat(dfs, ignore_index=True)
+        del dfs
+
+        trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
+        trades.loc[:, 'cost'] = trades['price'] * trades['amount']
+        for col in DEFAULT_TRADES_COLUMNS:
+            if col not in trades.columns:
+                trades.loc[:, col] = ''
+        trades = trades[DEFAULT_TRADES_COLUMNS]
+        trades = trades_convert_types(trades)
+
+        trades_df = trades_df_remove_duplicates(trades)
+        del trades
+        logger.info(f"{pair}: {len(trades_df)} trades, from "
+                    f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
+                    f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
+
+        data_handler.trades_store(pair, trades_df)

freqtrade/data/dataprovider.py

@@ -311,11 +311,13 @@ class DataProvider:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))
 
-            # It is not necessary to add the training candles, as they
-            # were already added at the beginning of the backtest.
-            startup_candles = self.get_required_startup(str(timeframe), False)
+            startup_candles = self.get_required_startup(str(timeframe))
             tf_seconds = timeframe_to_seconds(str(timeframe))
             timerange.subtract_start(tf_seconds * startup_candles)
+
+            logger.info(f"Loading data for {pair} {timeframe} "
+                        f"from {timerange.start_fmt} to {timerange.stop_fmt}")
+
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
                 timeframe=timeframe,
@@ -327,7 +329,7 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
-    def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int:
+    def get_required_startup(self, timeframe: str) -> int:
         freqai_config = self._config.get('freqai', {})
         if not freqai_config.get('enabled', False):
             return self._config.get('startup_candle_count', 0)
@@ -337,12 +339,11 @@ class DataProvider:
             # make sure the startupcandles is at least the set maximum indicator periods
             self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
             tf_seconds = timeframe_to_seconds(timeframe)
-            train_candles = 0
-            if add_train_candles:
-                train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
+            train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
             total_candles = int(self._config['startup_candle_count'] + train_candles)
-            logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
-            return total_candles
+            logger.info(
+                f'Increasing startup_candle_count for freqai on {timeframe} to {total_candles}')
+        return total_candles
 
     def get_pair_dataframe(
         self,

freqtrade/data/entryexitanalysis.py

@@ -211,8 +211,9 @@ def prepare_results(analysed_trades, stratname,
                     timerange=None):
     res_df = pd.DataFrame()
     for pair, trades in analysed_trades[stratname].items():
-        trades.dropna(subset=['close_date'], inplace=True)
-        res_df = pd.concat([res_df, trades], ignore_index=True)
+        if (trades.shape[0] > 0):
+            trades.dropna(subset=['close_date'], inplace=True)
+            res_df = pd.concat([res_df, trades], ignore_index=True)
 
     res_df = _select_rows_within_dates(res_df, timerange)
 

freqtrade/data/history/history_utils.py

@@ -8,7 +8,7 @@ from pandas import DataFrame, concat
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS,
-                                 DL_DATA_TIMEFRAMES, Config)
+                                 DL_DATA_TIMEFRAMES, DOCS_LINK, Config)
 from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv,
                                       ohlcv_to_dataframe, trades_df_remove_duplicates,
                                       trades_list_to_df)
@@ -18,8 +18,8 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
 from freqtrade.util import dt_ts, format_ms_time
-from freqtrade.util.binance_mig import migrate_binance_futures_data
 from freqtrade.util.datetime_helpers import dt_now
+from freqtrade.util.migrations import migrate_data
 
 
 logger = logging.getLogger(__name__)
@@ -311,15 +311,19 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             # Predefined candletype (and timeframe) depending on exchange
             # Downloads what is necessary to backtest based on futures data.
             tf_mark = exchange.get_option('mark_ohlcv_timeframe')
+            tf_funding_rate = exchange.get_option('funding_fee_timeframe')
+
             fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
             # All exchanges need FundingRate for futures trading.
             # The timeframe is aligned to the mark-price timeframe.
-            for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
+            combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark))
+            for candle_type_f, tf in combs:
+                logger.debug(f'Downloading pair {pair}, {candle_type_f}, interval {tf}.')
                 _download_pair_history(pair=pair, process=process,
                                        datadir=datadir, exchange=exchange,
                                        timerange=timerange, data_handler=data_handler,
-                                       timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
-                                       candle_type=funding_candle_type,
+                                       timeframe=str(tf), new_pairs_days=new_pairs_days,
+                                       candle_type=candle_type_f,
                                        erase=erase, prepend=prepend)
 
     return pairs_not_available
@@ -500,6 +504,12 @@ def download_data_main(config: Config) -> None:
     logger.info(f"About to download pairs: {expanded_pairs}, "
                 f"intervals: {config['timeframes']} to {config['datadir']}")
 
+    if len(expanded_pairs) == 0:
+        logger.warning(
+            "No pairs available for download. "
+            "Please make sure you're using the correct Pair naming for your selected trade mode. \n"
+            f"More info: {DOCS_LINK}/bot-basics/#pair-naming")
+
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
 
@@ -527,7 +537,7 @@ def download_data_main(config: Config) -> None:
                     "Please use `--dl-trades` instead for this exchange "
                     "(will unfortunately take a long time)."
                     )
-            migrate_binance_futures_data(config)
+            migrate_data(config, exchange)
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange,

freqtrade/data/history/idatahandler.py

@@ -94,21 +94,22 @@ class IDataHandler(ABC):
         """
 
     def ohlcv_data_min_max(self, pair: str, timeframe: str,
-                           candle_type: CandleType) -> Tuple[datetime, datetime]:
+                           candle_type: CandleType) -> Tuple[datetime, datetime, int]:
         """
         Returns the min and max timestamp for the given pair and timeframe.
         :param pair: Pair to get min/max for
         :param timeframe: Timeframe to get min/max for
         :param candle_type: Any of the enum CandleType (must match trading mode!)
-        :return: (min, max)
+        :return: (min, max, len)
         """
-        data = self._ohlcv_load(pair, timeframe, None, candle_type)
-        if data.empty:
+        df = self._ohlcv_load(pair, timeframe, None, candle_type)
+        if df.empty:
             return (
                 datetime.fromtimestamp(0, tz=timezone.utc),
-                datetime.fromtimestamp(0, tz=timezone.utc)
+                datetime.fromtimestamp(0, tz=timezone.utc),
+                0,
             )
-        return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
+        return df.iloc[0]['date'].to_pydatetime(), df.iloc[-1]['date'].to_pydatetime(), len(df)
 
     @abstractmethod
     def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -403,6 +404,34 @@ class IDataHandler(ABC):
             return
         file_old.rename(file_new)
 
+    def fix_funding_fee_timeframe(self, ff_timeframe: str):
+        """
+        Temporary method to migrate data from old funding fee timeframe to the correct timeframe
+        Applies to bybit and okx, where funding-fee and mark candles have different timeframes.
+        """
+        paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES)
+        funding_rate_combs = [
+            f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe
+        ]
+
+        if funding_rate_combs:
+            logger.warning(
+                f'Migrating {len(funding_rate_combs)} funding fees to correct timeframe.')
+
+        for pair, timeframe, candletype in funding_rate_combs:
+            old_name = self._pair_data_filename(self._datadir, pair, timeframe, candletype)
+            new_name = self._pair_data_filename(self._datadir, pair, ff_timeframe, candletype)
+
+            if not Path(old_name).exists():
+                logger.warning(f'{old_name} does not exist, skipping.')
+                continue
+
+            if Path(new_name).exists():
+                logger.warning(f'{new_name} already exists, Removing.')
+                Path(new_name).unlink()
+
+            Path(old_name).rename(new_name)
+
 
 def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     """

freqtrade/data/metrics.py

@@ -61,10 +61,10 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     if len(trades) == 0:
         raise ValueError("Trade dataframe empty.")
-    from freqtrade.exchange import timeframe_to_minutes
-    timeframe_minutes = timeframe_to_minutes(timeframe)
+    from freqtrade.exchange import timeframe_to_resample_freq
+    timeframe_freq = timeframe_to_resample_freq(timeframe)
     # Resample to timeframe to make sure trades match candles
-    _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
+    _trades_sum = trades.resample(timeframe_freq, on='close_date'
                                   )[['profit_abs']].sum()
     df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
     # Set first value to 0
@@ -143,8 +143,10 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
         starting_balance=starting_balance
     )
 
-    idxmin = max_drawdown_df['drawdown_relative'].idxmax() if relative \
-        else max_drawdown_df['drawdown'].idxmin()
+    idxmin = (
+        max_drawdown_df['drawdown_relative'].idxmax()
+        if relative else max_drawdown_df['drawdown'].idxmin()
+    )
     if idxmin == 0:
         raise ValueError("No losing trade, therefore no drawdown.")
     high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
@@ -191,6 +193,9 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
     :param final_balance: Final balance to calculate CAGR against
     :return: CAGR
     """
+    if final_balance < 0:
+        # With leveraged trades, final_balance can become negative.
+        return 0
     return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
 
 

freqtrade/exchange/__init__.py

@@ -4,8 +4,8 @@ from freqtrade.exchange.common import remove_exchange_credentials, MAP_EXCHANGE_
 from freqtrade.exchange.exchange import Exchange
 # isort: on
 from freqtrade.exchange.binance import Binance
+from freqtrade.exchange.bitmart import Bitmart
 from freqtrade.exchange.bitpanda import Bitpanda
-from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bitvavo import Bitvavo
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
@@ -17,10 +17,11 @@ from freqtrade.exchange.exchange_utils import (ROUND_DOWN, ROUND_UP, amount_to_c
                                                market_is_active, price_to_precision,
                                                timeframe_to_minutes, timeframe_to_msecs,
                                                timeframe_to_next_date, timeframe_to_prev_date,
-                                               timeframe_to_seconds, validate_exchange)
+                                               timeframe_to_resample_freq, timeframe_to_seconds,
+                                               validate_exchange)
 from freqtrade.exchange.gate import Gate
 from freqtrade.exchange.hitbtc import Hitbtc
-from freqtrade.exchange.huobi import Huobi
+from freqtrade.exchange.htx import Htx
 from freqtrade.exchange.kraken import Kraken
 from freqtrade.exchange.kucoin import Kucoin
 from freqtrade.exchange.okx import Okx

freqtrade/exchange/binance.py

@@ -123,10 +123,14 @@ class Binance(Exchange):
 
     def funding_fee_cutoff(self, open_date: datetime):
         """
+        Funding fees are only charged at full hours (usually every 4-8h).
+        Therefore a trade opening at 10:00:01 will not be charged a funding fee until the next hour.
+        On binance, this cutoff is 15s.
+        https://github.com/freqtrade/freqtrade/pull/5779#discussion_r740175931
         :param open_date: The open date for a trade
-        :return: The cutoff open time for when a funding fee is charged
+        :return: True if the date falls on a full hour, False otherwise
         """
-        return open_date.minute > 0 or (open_date.minute == 0 and open_date.second > 15)
+        return open_date.minute == 0 and open_date.second < 15
 
     def dry_run_liquidation_price(
         self,

freqtrade/exchange/bitmart.py

@@ -0,0 +1,20 @@
+""" Bitmart exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Bitmart(Exchange):
+    """
+    Bitmart exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+    """
+
+    _ft_has: Dict = {
+        "stoploss_on_exchange": False,  # Bitmart API does not support stoploss orders
+        "ohlcv_candle_limit": 200,
+    }

freqtrade/exchange/bittrex.py

@@ -1,25 +0,0 @@
-""" Bittrex exchange subclass """
-import logging
-from typing import Dict
-
-from freqtrade.exchange import Exchange
-
-
-logger = logging.getLogger(__name__)
-
-
-class Bittrex(Exchange):
-    """
-    Bittrex exchange class. Contains adjustments needed for Freqtrade to work
-    with this exchange.
-    """
-
-    _ft_has: Dict = {
-        "ohlcv_candle_limit_per_timeframe": {
-            '1m': 1440,
-            '5m': 288,
-            '1h': 744,
-            '1d': 365,
-        },
-        "l2_limit_range": [1, 25, 500],
-    }

freqtrade/exchange/bybit.py

@@ -7,7 +7,7 @@ import ccxt
 
 from freqtrade.constants import BuySell
 from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode
-from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
+from freqtrade.exceptions import DDosProtection, ExchangeError, OperationalException, TemporaryError
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
 from freqtrade.util.datetime_helpers import dt_now, dt_ts
@@ -25,10 +25,12 @@ class Bybit(Exchange):
     officially supported by the Freqtrade development team. So some features
     may still not work as expected.
     """
+    unified_account = False
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
+        "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
     }
     _ft_has_futures: Dict = {
         "ohlcv_has_history": True,
@@ -81,9 +83,20 @@ class Bybit(Exchange):
         Must be overridden in child methods if required.
         """
         try:
-            if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
-                position_mode = self._api.set_position_mode(False)
-                self._log_exchange_response('set_position_mode', position_mode)
+            if not self._config['dry_run']:
+                if self.trading_mode == TradingMode.FUTURES:
+                    position_mode = self._api.set_position_mode(False)
+                    self._log_exchange_response('set_position_mode', position_mode)
+                is_unified = self._api.is_unified_enabled()
+                # Returns a tuple of bools, first for margin, second for Account
+                if is_unified and len(is_unified) > 1 and is_unified[1]:
+                    self.unified_account = True
+                    logger.info("Bybit: Unified account.")
+                    raise OperationalException("Bybit: Unified account is not supported. "
+                                               "Please use a standard (sub)account.")
+                else:
+                    self.unified_account = False
+                    logger.info("Bybit: Standard account.")
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
@@ -202,8 +215,11 @@ class Bybit(Exchange):
         """
         # Bybit does not provide "applied" funding fees per position.
         if self.trading_mode == TradingMode.FUTURES:
-            return self._fetch_and_calculate_funding_fees(
-                    pair, amount, is_short, open_date)
+            try:
+                return self._fetch_and_calculate_funding_fees(
+                        pair, amount, is_short, open_date)
+            except ExchangeError:
+                logger.warning(f"Could not update funding fees for {pair}.")
         return 0.0
 
     def fetch_orders(self, pair: str, since: datetime, params: Optional[Dict] = None) -> List[Dict]:

freqtrade/exchange/common.py

@@ -2,7 +2,7 @@ import asyncio
 import logging
 import time
 from functools import wraps
-from typing import Any, Callable, Optional, TypeVar, cast, overload
+from typing import Any, Callable, Dict, List, Optional, TypeVar, cast, overload
 
 from freqtrade.constants import ExchangeConfig
 from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
@@ -48,27 +48,29 @@ MAP_EXCHANGE_CHILDCLASS = {
     'binanceusdm': 'binance',
     'okex': 'okx',
     'gateio': 'gate',
+    'huboi': 'htx',
 }
 
 SUPPORTED_EXCHANGES = [
     'binance',
-    'bittrex',
+    'bitmart',
     'gate',
-    'huobi',
+    'htx',
     'kraken',
     'okx',
 ]
 
-EXCHANGE_HAS_REQUIRED = [
+# either the main, or replacement methods (array) is required
+EXCHANGE_HAS_REQUIRED: Dict[str, List[str]] = {
     # Required / private
-    'fetchOrder',
-    'cancelOrder',
-    'createOrder',
-    'fetchBalance',
+    'fetchOrder': ['fetchOpenOrder', 'fetchClosedOrder'],
+    'cancelOrder': [],
+    'createOrder': [],
+    'fetchBalance': [],
 
     # Public endpoints
-    'fetchOHLCV',
-]
+    'fetchOHLCV': [],
+}
 
 EXCHANGE_HAS_OPTIONAL = [
     # Private
@@ -85,6 +87,7 @@ EXCHANGE_HAS_OPTIONAL = [
     # 'fetchPositions',  # Futures trading
     # 'fetchLeverageTiers',  # Futures initialization
     # 'fetchMarketLeverageTiers',  # Futures initialization
+    # 'fetchOpenOrder', 'fetchClosedOrder',  # replacement for fetchOrder
     # 'fetchOpenOrders', 'fetchClosedOrders',  # 'fetchOrders',  # Refinding balance...
 ]
 

freqtrade/exchange/exchange.py

@@ -23,7 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN
                                  BuySell, Config, EntryExit, ExchangeConfig,
                                  ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode
+from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, RunMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
@@ -43,6 +43,7 @@ from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.util import dt_from_ts, dt_now
 from freqtrade.util.datetime_helpers import dt_humanize, dt_ts
+from freqtrade.util.periodic_cache import PeriodicCache
 
 
 logger = logging.getLogger(__name__)
@@ -80,6 +81,7 @@ class Exchange:
         "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
         "mark_ohlcv_price": "mark",
         "mark_ohlcv_timeframe": "8h",
+        "funding_fee_timeframe": "8h",
         "ccxt_futures_name": "swap",
         "needs_trading_fees": False,  # use fetch_trading_fees to cache fees
         "order_props_in_contracts": ['amount', 'filled', 'remaining'],
@@ -121,14 +123,16 @@ class Exchange:
         # Cache for 10 minutes ...
         self._cache_lock = Lock()
         self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10)
-        # Cache values for 1800 to avoid frequent polling of the exchange for prices
+        # Cache values for 300 to avoid frequent polling of the exchange for prices
         # Caching only applies to RPC methods, so prices for open trades are still
         # refreshed once every iteration.
-        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
-        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800)
+        # Shouldn't be too high either, as it'll freeze UI updates in case of open orders.
+        self._exit_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
+        self._entry_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
 
         # Holds candles
         self._klines: Dict[PairWithTimeframe, DataFrame] = {}
+        self._expiring_candle_cache: Dict[Tuple[str, int], PeriodicCache] = {}
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
@@ -319,10 +323,11 @@ class Exchange:
         """
         pass
 
-    def _log_exchange_response(self, endpoint, response) -> None:
+    def _log_exchange_response(self, endpoint: str, response, *, add_info=None) -> None:
         """ Log exchange responses """
         if self.log_responses:
-            logger.info(f"API {endpoint}: {response}")
+            add_info_str = "" if add_info is None else f" {add_info}: "
+            logger.info(f"API {endpoint}: {add_info_str}{response}")
 
     def ohlcv_candle_limit(
             self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
@@ -330,6 +335,7 @@ class Exchange:
         Exchange ohlcv candle limit
         Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits
         per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit
+        TODO: this is most likely no longer needed since only bittrex needed this.
         :param timeframe: Timeframe to check
         :param candle_type: Candle-type
         :param since_ms: Starting timestamp
@@ -486,11 +492,14 @@ class Exchange:
         except ccxt.BaseError:
             logger.exception('Unable to initialize markets.')
 
-    def reload_markets(self) -> None:
+    def reload_markets(self, force: bool = False) -> None:
         """Reload markets both sync and async if refresh interval has passed """
         # Check whether markets have to be reloaded
-        if (self._last_markets_refresh > 0) and (
-                self._last_markets_refresh + self.markets_refresh_interval > dt_ts()):
+        if (
+            not force
+            and self._last_markets_refresh > 0
+            and (self._last_markets_refresh + self.markets_refresh_interval > dt_ts())
+        ):
             return None
         logger.debug("Performing scheduled market reload..")
         try:
@@ -588,7 +597,11 @@ class Exchange:
             raise OperationalException(
                 f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}")
 
-        if timeframe and timeframe_to_minutes(timeframe) < 1:
+        if (
+            timeframe
+            and self._config['runmode'] != RunMode.UTIL_EXCHANGE
+            and timeframe_to_minutes(timeframe) < 1
+        ):
             raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.")
 
     def validate_ordertypes(self, order_types: Dict) -> None:
@@ -1228,16 +1241,16 @@ class Exchange:
             return order
         except ccxt.InsufficientFunds as e:
             raise InsufficientFundsError(
-                f'Insufficient funds to create {ordertype} sell order on market {pair}. '
-                f'Tried to sell amount {amount} at rate {limit_rate}. '
-                f'Message: {e}') from e
-        except ccxt.InvalidOrder as e:
+                f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
+                f'Tried to {side} amount {amount} at rate {limit_rate} with '
+                f'stop-price {stop_price_norm}. Message: {e}') from e
+        except (ccxt.InvalidOrder, ccxt.BadRequest, ccxt.OperationRejected) as e:
             # Errors:
             # `Order would trigger immediately.`
             raise InvalidOrderException(
-                f'Could not create {ordertype} sell order on market {pair}. '
-                f'Tried to sell amount {amount} at rate {limit_rate}. '
-                f'Message: {e}') from e
+                f'Could not create {ordertype} {side} order on market {pair}. '
+                f'Tried to {side} amount {amount} at rate {limit_rate} with '
+                f'stop-price {stop_price_norm}. Message: {e}') from e
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
@@ -1247,11 +1260,43 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def fetch_order_emulated(self, order_id: str, pair: str, params: Dict) -> Dict:
+        """
+        Emulated fetch_order if the exchange doesn't support fetch_order, but requires separate
+        calls for open and closed orders.
+        """
+        try:
+            order = self._api.fetch_open_order(order_id, pair, params=params)
+            self._log_exchange_response('fetch_open_order', order)
+            order = self._order_contracts_to_amount(order)
+            return order
+        except ccxt.OrderNotFound:
+            try:
+                order = self._api.fetch_closed_order(order_id, pair, params=params)
+                self._log_exchange_response('fetch_closed_order', order)
+                order = self._order_contracts_to_amount(order)
+                return order
+            except ccxt.OrderNotFound as e:
+                raise RetryableOrderError(
+                    f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.InvalidOrder as e:
+            raise InvalidOrderException(
+                f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e
+        except ccxt.DDoSProtection as e:
+            raise DDosProtection(e) from e
+        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
+            raise TemporaryError(
+                f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e
+        except ccxt.BaseError as e:
+            raise OperationalException(e) from e
+
     @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
     def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
         if self._config['dry_run']:
             return self.fetch_dry_run_order(order_id)
         try:
+            if not self.exchange_has('fetchOrder'):
+                return self.fetch_order_emulated(order_id, pair, params)
             order = self._api.fetch_order(order_id, pair, params=params)
             self._log_exchange_response('fetch_order', order)
             order = self._order_contracts_to_amount(order)
@@ -1380,7 +1425,7 @@ class Exchange:
             order = self.fetch_stoploss_order(order_id, pair)
         except InvalidOrderException:
             logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
-            order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
+            order = {'id': order_id, 'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
 
         return order
 
@@ -1496,8 +1541,9 @@ class Exchange:
     @retrier
     def fetch_bids_asks(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict:
         """
+        :param symbols: List of symbols to fetch
         :param cached: Allow cached result
-        :return: fetch_tickers result
+        :return: fetch_bids_asks result
         """
         if not self.exchange_has('fetchBidsAsks'):
             return {}
@@ -1546,6 +1592,12 @@ class Exchange:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching tickers in batch. '
                 f'Message: {e}') from e
+        except ccxt.BadSymbol as e:
+            logger.warning(f"Could not load tickers due to {e.__class__.__name__}. Message: {e} ."
+                           "Reloading markets.")
+            self.reload_markets(True)
+            # Re-raise exception to repeat the call.
+            raise TemporaryError from e
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
         except (ccxt.NetworkError, ccxt.ExchangeError) as e:
@@ -1954,7 +2006,7 @@ class Exchange:
 
             results = await asyncio.gather(*input_coro, return_exceptions=True)
             for res in results:
-                if isinstance(res, Exception):
+                if isinstance(res, BaseException):
                     logger.warning(f"Async code raised an exception: {repr(res)}")
                     if raise_:
                         raise
@@ -2106,6 +2158,39 @@ class Exchange:
 
         return results_df
 
+    def refresh_ohlcv_with_cache(
+        self,
+        pairs: List[PairWithTimeframe],
+        since_ms: int
+    ) -> Dict[PairWithTimeframe, DataFrame]:
+        """
+        Refresh ohlcv data for all pairs in needed_pairs if necessary.
+        Caches data with expiring per timeframe.
+        Should only be used for pairlists which need "on time" expirarion, and no longer cache.
+        """
+
+        timeframes = {p[1] for p in pairs}
+        for timeframe in timeframes:
+            if (timeframe, since_ms) not in self._expiring_candle_cache:
+                timeframe_in_sec = timeframe_to_seconds(timeframe)
+                # Initialise cache
+                self._expiring_candle_cache[(timeframe, since_ms)] = PeriodicCache(
+                    ttl=timeframe_in_sec, maxsize=1000)
+
+        # Get candles from cache
+        candles = {
+            c: self._expiring_candle_cache[(c[1], since_ms)].get(c, None) for c in pairs
+            if c in self._expiring_candle_cache[(c[1], since_ms)]
+        }
+        pairs_to_download = [p for p in pairs if p not in candles]
+        if pairs_to_download:
+            candles = self.refresh_latest_ohlcv(
+                pairs_to_download, since_ms=since_ms, cache=False
+            )
+            for c, val in candles.items():
+                self._expiring_candle_cache[(c[1], since_ms)][c] = val
+        return candles
+
     def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
         # Timeframe in seconds
         interval_in_sec = timeframe_to_seconds(timeframe)
@@ -2202,13 +2287,13 @@ class Exchange:
     @retrier_async
     async def _async_fetch_trades(self, pair: str,
                                   since: Optional[int] = None,
-                                  params: Optional[dict] = None) -> List[List]:
+                                  params: Optional[dict] = None) -> Tuple[List[List], Any]:
         """
         Asyncronously gets trade history using fetch_trades.
         Handles exchange errors, does one call to the exchange.
         :param pair: Pair to fetch trade data for
         :param since: Since as integer timestamp in milliseconds
-        returns: List of dicts containing trades
+        returns: List of dicts containing trades, the next iteration value (new "since" or trade_id)
         """
         try:
             # fetch trades asynchronously
@@ -2223,7 +2308,8 @@ class Exchange:
                 )
                 trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
             trades = self._trades_contracts_to_amount(trades)
-            return trades_dict_to_list(trades)
+            pagination_value = self._get_trade_pagination_next_value(trades)
+            return trades_dict_to_list(trades), pagination_value
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching historical trade data.'
@@ -2236,6 +2322,25 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e
 
+    def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool:
+        """
+        Verify trade-pagination id is valid.
+        Workaround for odd Kraken issue where ID is sometimes wrong.
+        """
+        return True
+
+    def _get_trade_pagination_next_value(self, trades: List[Dict]):
+        """
+        Extract pagination id for the next "from_id" value
+        Applies only to fetch_trade_history by id.
+        """
+        if not trades:
+            return None
+        if self._trades_pagination == 'id':
+            return trades[-1].get('id')
+        else:
+            return trades[-1].get('timestamp')
+
     async def _async_get_trade_history_id(self, pair: str,
                                           until: int,
                                           since: Optional[int] = None,
@@ -2251,34 +2356,37 @@ class Exchange:
         """
 
         trades: List[List] = []
+        # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
+        # DEFAULT_TRADES_COLUMNS: 1 -> id
+        has_overlap = self._ft_has.get('trades_pagination_overlap', True)
+        # Skip last trade by default since its the key for the next call
+        x = slice(None, -1) if has_overlap else slice(None)
 
-        if not from_id:
+        if not from_id or not self._valid_trade_pagination_id(pair, from_id):
             # Fetch first elements using timebased method to get an ID to paginate on
             # Depending on the Exchange, this can introduce a drift at the start of the interval
             # of up to an hour.
             # e.g. Binance returns the "last 1000" candles within a 1h time interval
             # - so we will miss the first trades.
-            t = await self._async_fetch_trades(pair, since=since)
-            # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
-            # DEFAULT_TRADES_COLUMNS: 1 -> id
-            from_id = t[-1][1]
-            trades.extend(t[:-1])
+            t, from_id = await self._async_fetch_trades(pair, since=since)
+            trades.extend(t[x])
         while True:
             try:
-                t = await self._async_fetch_trades(pair,
-                                                   params={self._trades_pagination_arg: from_id})
+                t, from_id_next = await self._async_fetch_trades(
+                    pair, params={self._trades_pagination_arg: from_id})
                 if t:
-                    # Skip last id since its the key for the next call
-                    trades.extend(t[:-1])
-                    if from_id == t[-1][1] or t[-1][0] > until:
+                    trades.extend(t[x])
+                    if from_id == from_id_next or t[-1][0] > until:
                         logger.debug(f"Stopping because from_id did not change. "
                                      f"Reached {t[-1][0]} > {until}")
                         # Reached the end of the defined-download period - add last trade as well.
-                        trades.extend(t[-1:])
+                        if has_overlap:
+                            trades.extend(t[-1:])
                         break
 
-                    from_id = t[-1][1]
+                    from_id = from_id_next
                 else:
+                    logger.debug("Stopping as no more trades were returned.")
                     break
             except asyncio.CancelledError:
                 logger.debug("Async operation Interrupted, breaking trades DL loop.")
@@ -2302,16 +2410,22 @@ class Exchange:
         # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
             try:
-                t = await self._async_fetch_trades(pair, since=since)
+                t, since_next = await self._async_fetch_trades(pair, since=since)
                 if t:
-                    since = t[-1][0]
+                    # No more trades to download available at the exchange,
+                    # So we repeatedly get the same trade over and over again.
+                    if since == since_next and len(t) == 1:
+                        logger.debug("Stopping because no more trades are available.")
+                        break
+                    since = since_next
                     trades.extend(t)
                     # Reached the end of the defined-download period
-                    if until and t[-1][0] > until:
+                    if until and since_next > until:
                         logger.debug(
-                            f"Stopping because until was reached. {t[-1][0]} > {until}")
+                            f"Stopping because until was reached. {since_next} > {until}")
                         break
                 else:
+                    logger.debug("Stopping as no more trades were returned.")
                     break
             except asyncio.CancelledError:
                 logger.debug("Async operation Interrupted, breaking trades DL loop.")
@@ -2396,6 +2510,8 @@ class Exchange:
                 symbol=pair,
                 since=since
             )
+            self._log_exchange_response('funding_history', funding_history,
+                                        add_info=f"pair: {pair}, since: {since}")
             return sum(fee['amount'] for fee in funding_history)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
@@ -2636,7 +2752,7 @@ class Exchange:
             self._log_exchange_response('set_leverage', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except (ccxt.BadRequest, ccxt.InsufficientFunds) as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected, ccxt.InsufficientFunds) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
@@ -2653,12 +2769,14 @@ class Exchange:
         """
         return 0.0
 
-    def funding_fee_cutoff(self, open_date: datetime):
+    def funding_fee_cutoff(self, open_date: datetime) -> bool:
         """
+        Funding fees are only charged at full hours (usually every 4-8h).
+        Therefore a trade opening at 10:00:01 will not be charged a funding fee until the next hour.
         :param open_date: The open date for a trade
-        :return: The cutoff open time for when a funding fee is charged
+        :return: True if the date falls on a full hour, False otherwise
         """
-        return open_date.minute > 0 or open_date.second > 0
+        return open_date.minute == 0 and open_date.second == 0
 
     @retrier
     def set_margin_mode(self, pair: str, margin_mode: MarginMode, accept_fail: bool = False,
@@ -2676,7 +2794,7 @@ class Exchange:
             self._log_exchange_response('set_margin_mode', res)
         except ccxt.DDoSProtection as e:
             raise DDosProtection(e) from e
-        except ccxt.BadRequest as e:
+        except (ccxt.BadRequest, ccxt.OperationRejected) as e:
             if not accept_fail:
                 raise TemporaryError(
                     f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e
@@ -2706,23 +2824,23 @@ class Exchange:
         """
 
         if self.funding_fee_cutoff(open_date):
-            open_date += timedelta(hours=1)
+            # Shift back to 1h candle to avoid missing funding fees
+            # Only really relevant for trades very close to the full hour
+            open_date = timeframe_to_prev_date('1h', open_date)
         timeframe = self._ft_has['mark_ohlcv_timeframe']
-        timeframe_ff = self._ft_has.get('funding_fee_timeframe',
-                                        self._ft_has['mark_ohlcv_timeframe'])
+        timeframe_ff = self._ft_has['funding_fee_timeframe']
+        mark_price_type = CandleType.from_string(self._ft_has["mark_ohlcv_price"])
 
         if not close_date:
             close_date = datetime.now(timezone.utc)
-        open_timestamp = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
-        # close_timestamp = int(close_date.timestamp()) * 1000
-
-        mark_comb: PairWithTimeframe = (
-            pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"]))
+        since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000
 
+        mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type)
         funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE)
+
         candle_histories = self.refresh_latest_ohlcv(
             [mark_comb, funding_comb],
-            since_ms=open_timestamp,
+            since_ms=since_ms,
             cache=False,
             drop_incomplete=False,
         )
@@ -2733,8 +2851,7 @@ class Exchange:
         except KeyError:
             raise ExchangeError("Could not find funding rates.") from None
 
-        funding_mark_rates = self.combine_funding_and_mark(
-            funding_rates=funding_rates, mark_rates=mark_rates)
+        funding_mark_rates = self.combine_funding_and_mark(funding_rates, mark_rates)
 
         return self.calculate_funding_fees(
             funding_mark_rates,
@@ -2781,7 +2898,7 @@ class Exchange:
         amount: float,
         is_short: bool,
         open_date: datetime,
-        close_date: Optional[datetime] = None,
+        close_date: datetime,
         time_in_ratio: Optional[float] = None
     ) -> float:
         """
@@ -2797,8 +2914,8 @@ class Exchange:
         fees: float = 0
 
         if not df.empty:
-            df = df[(df['date'] >= open_date) & (df['date'] <= close_date)]
-            fees = sum(df['open_fund'] * df['open_mark'] * amount)
+            df1 = df[(df['date'] >= open_date) & (df['date'] <= close_date)]
+            fees = sum(df1['open_fund'] * df1['open_mark'] * amount)
 
         # Negate fees for longs as funding_fees expects it this way based on live endpoints.
         return fees if is_short else -fees
@@ -2813,17 +2930,19 @@ class Exchange:
         :param amount: Trade amount
         :param open_date: Open date of the trade
         :return: funding fee since open_date
-        :raises: ExchangeError if something goes wrong.
         """
         if self.trading_mode == TradingMode.FUTURES:
-            if self._config['dry_run']:
-                funding_fees = self._fetch_and_calculate_funding_fees(
-                    pair, amount, is_short, open_date)
-            else:
-                funding_fees = self._get_funding_fees_from_exchange(pair, open_date)
-            return funding_fees
-        else:
-            return 0.0
+            try:
+                if self._config['dry_run']:
+                    funding_fees = self._fetch_and_calculate_funding_fees(
+                        pair, amount, is_short, open_date)
+                else:
+                    funding_fees = self._get_funding_fees_from_exchange(pair, open_date)
+                return funding_fees
+            except ExchangeError:
+                logger.warning(f"Could not update funding fees for {pair}.")
+
+        return 0.0
 
     def get_liquidation_price(
         self,

freqtrade/exchange/exchange_utils.py

@@ -40,21 +40,34 @@ def available_exchanges(ccxt_module: Optional[CcxtModuleType] = None) -> List[st
 
 
 def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    """
+    returns: can_use, reason
+        with Reason including both missing and missing_opt
+    """
     ex_mod = getattr(ccxt, exchange.lower())()
+    result = True
+    reason = ''
     if not ex_mod or not ex_mod.has:
         return False, ''
-    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    missing = [
+        k for k, v in EXCHANGE_HAS_REQUIRED.items()
+        if ex_mod.has.get(k) is not True
+        and not (all(ex_mod.has.get(x) for x in v))
+    ]
     if missing:
-        return False, f"missing: {', '.join(missing)}"
+        result = False
+        reason += f"missing: {', '.join(missing)}"
 
     missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
 
     if exchange.lower() in BAD_EXCHANGES:
-        return False, BAD_EXCHANGES.get(exchange.lower(), '')
-    if missing_opt:
-        return True, f"missing opt: {', '.join(missing_opt)}"
+        result = False
+        reason = BAD_EXCHANGES.get(exchange.lower(), '')
 
-    return True, ''
+    if missing_opt:
+        reason += f"{'. ' if reason else ''}missing opt: {', '.join(missing_opt)}. "
+
+    return result, reason
 
 
 def _build_exchange_list_entry(
@@ -118,6 +131,27 @@ def timeframe_to_msecs(timeframe: str) -> int:
     return ccxt.Exchange.parse_timeframe(timeframe) * 1000
 
 
+def timeframe_to_resample_freq(timeframe: str) -> str:
+    """
+    Translates the timeframe interval value written in the human readable
+    form ('1m', '5m', '1h', '1d', '1w', etc.) to the resample frequency
+    used by pandas ('1T', '5T', '1H', '1D', '1W', etc.)
+    """
+    if timeframe == '1y':
+        return '1YS'
+    timeframe_seconds = timeframe_to_seconds(timeframe)
+    timeframe_minutes = timeframe_seconds // 60
+    resample_interval = f'{timeframe_seconds}s'
+    if 10000 < timeframe_minutes < 43200:
+        resample_interval = '1W-MON'
+    elif timeframe_minutes >= 43200 and timeframe_minutes < 525600:
+        # Monthly candles need special treatment to stick to the 1st of the month
+        resample_interval = f'{timeframe}S'
+    elif timeframe_minutes > 43200:
+        resample_interval = timeframe
+    return resample_interval
+
+
 def timeframe_to_prev_date(timeframe: str, date: Optional[datetime] = None) -> datetime:
     """
     Use Timeframe and determine the candle start date for this date.

freqtrade/exchange/htx.py

@@ -1,4 +1,4 @@
-""" Huobi exchange subclass """
+""" HTX exchange subclass """
 import logging
 from typing import Dict
 
@@ -9,9 +9,9 @@ from freqtrade.exchange import Exchange
 logger = logging.getLogger(__name__)
 
 
-class Huobi(Exchange):
+class Htx(Exchange):
     """
-    Huobi exchange class. Contains adjustments needed for Freqtrade to work
+    HTX exchange class. Contains adjustments needed for Freqtrade to work
     with this exchange.
     """
 

freqtrade/exchange/kraken.py

@@ -8,11 +8,9 @@ from pandas import DataFrame
 
 from freqtrade.constants import BuySell
 from freqtrade.enums import MarginMode, TradingMode
-from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
-                                  OperationalException, TemporaryError)
+from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
-from freqtrade.exchange.exchange_utils import ROUND_DOWN, ROUND_UP
 from freqtrade.exchange.types import Tickers
 
 
@@ -24,12 +22,15 @@ class Kraken(Exchange):
     _params: Dict = {"trading_agreement": "agree"}
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
-        "stop_price_param": "stopPrice",
-        "stop_price_prop": "stopPrice",
+        "stop_price_param": "stopLossPrice",
+        "stop_price_prop": "stopLossPrice",
+        "stoploss_order_types": {"limit": "limit", "market": "market"},
+        "order_time_in_force": ["GTC", "IOC", "PO"],
         "ohlcv_candle_limit": 720,
         "ohlcv_has_history": False,
         "trades_pagination": "id",
         "trades_pagination_arg": "since",
+        "trades_pagination_overlap": False,
         "mark_ohlcv_timeframe": "4h",
     }
 
@@ -89,75 +90,6 @@ class Kraken(Exchange):
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
-        """
-        Verify stop_loss against stoploss-order value (limit or price)
-        Returns True if adjustment is necessary.
-        """
-        return (order['type'] in ('stop-loss', 'stop-loss-limit') and (
-                (side == "sell" and stop_loss > float(order['price'])) or
-                (side == "buy" and stop_loss < float(order['price']))
-                ))
-
-    @retrier(retries=0)
-    def create_stoploss(self, pair: str, amount: float, stop_price: float,
-                        order_types: Dict, side: BuySell, leverage: float) -> Dict:
-        """
-        Creates a stoploss market order.
-        Stoploss market orders is the only stoploss type supported by kraken.
-        TODO: investigate if this can be combined with generic implementation
-              (careful, prices are reversed)
-        """
-        params = self._params.copy()
-        if self.trading_mode == TradingMode.FUTURES:
-            params.update({'reduceOnly': True})
-
-        round_mode = ROUND_DOWN if side == 'buy' else ROUND_UP
-        if order_types.get('stoploss', 'market') == 'limit':
-            ordertype = "stop-loss-limit"
-            limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
-            if side == "sell":
-                limit_rate = stop_price * limit_price_pct
-            else:
-                limit_rate = stop_price * (2 - limit_price_pct)
-            params['price2'] = self.price_to_precision(pair, limit_rate, rounding_mode=round_mode)
-        else:
-            ordertype = "stop-loss"
-
-        stop_price = self.price_to_precision(pair, stop_price, rounding_mode=round_mode)
-
-        if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(
-                pair, ordertype, side, amount, stop_price, leverage, stop_loss=True)
-            return dry_order
-
-        try:
-            amount = self.amount_to_precision(pair, amount)
-
-            order = self._api.create_order(symbol=pair, type=ordertype, side=side,
-                                           amount=amount, price=stop_price, params=params)
-            self._log_exchange_response('create_stoploss_order', order)
-            logger.info('stoploss order added for %s. '
-                        'stop price: %s.', pair, stop_price)
-            return order
-        except ccxt.InsufficientFunds as e:
-            raise InsufficientFundsError(
-                f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.InvalidOrder as e:
-            raise InvalidOrderException(
-                f'Could not create {ordertype} {side} order on market {pair}. '
-                f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
-                f'Message: {e}') from e
-        except ccxt.DDoSProtection as e:
-            raise DDosProtection(e) from e
-        except (ccxt.NetworkError, ccxt.ExchangeError) as e:
-            raise TemporaryError(
-                f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
-        except ccxt.BaseError as e:
-            raise OperationalException(e) from e
-
     def _set_leverage(
         self,
         leverage: float,
@@ -187,6 +119,9 @@ class Kraken(Exchange):
         )
         if leverage > 1.0:
             params['leverage'] = round(leverage)
+        if time_in_force == 'PO':
+            params.pop('timeInForce', None)
+            params['postOnly'] = True
         return params
 
     def calculate_funding_fees(
@@ -195,7 +130,7 @@ class Kraken(Exchange):
         amount: float,
         is_short: bool,
         open_date: datetime,
-        close_date: Optional[datetime] = None,
+        close_date: datetime,
         time_in_ratio: Optional[float] = None
     ) -> float:
         """
@@ -223,18 +158,30 @@ class Kraken(Exchange):
 
         return fees if is_short else -fees
 
-    def _trades_contracts_to_amount(self, trades: List) -> List:
+    def _get_trade_pagination_next_value(self, trades: List[Dict]):
         """
-        Fix "last" id issue for kraken data downloads
-        This whole override can probably be removed once the following
-        issue is closed in ccxt: https://github.com/ccxt/ccxt/issues/15827
+        Extract pagination id for the next "from_id" value
+        Applies only to fetch_trade_history by id.
         """
-        super()._trades_contracts_to_amount(trades)
-        if (
-            len(trades) > 0
-            and isinstance(trades[-1].get('info'), list)
-            and len(trades[-1].get('info', [])) > 7
-        ):
+        if len(trades) > 0:
+            if (
+                isinstance(trades[-1].get('info'), list)
+                and len(trades[-1].get('info', [])) > 7
+            ):
+                # Trade response's "last" value.
+                return trades[-1].get('info', [])[-1]
+            # Fall back to timestamp if info is somehow empty.
+            return trades[-1].get('timestamp')
+        return None
 
-            trades[-1]['id'] = trades[-1].get('info', [])[-1]
-        return trades
+    def _valid_trade_pagination_id(self, pair: str, from_id: str) -> bool:
+        """
+        Verify trade-pagination id is valid.
+        Workaround for odd Kraken issue where ID is sometimes wrong.
+        """
+        # Regular id's are in timestamp format 1705443695120072285
+        # If the id is smaller than 19 characters, it's not a valid timestamp.
+        if len(from_id) >= 19:
+            return True
+        logger.debug(f"{pair} - trade-pagination id is not valid. Fallback to timestamp.")
+        return False

freqtrade/exchange/okx.py

@@ -228,7 +228,7 @@ class Okx(Exchange):
                 f'StoplossOrder not found (pair: {pair} id: {order_id}).')
 
     def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
-        if order['type'] == 'stop':
+        if order.get('type', '') == 'stop':
             return safe_value_fallback2(order, order, 'id_stop', 'id')
         return order['id']
 

freqtrade/freqai/RL/BaseEnvironment.py

@@ -159,7 +159,7 @@ class BaseEnvironment(gym.Env):
         function is designed for tracking incremented objects,
         events, actions inside the training environment.
         For example, a user can call this to track the
-        frequency of occurence of an `is_valid` call in
+        frequency of occurrence of an `is_valid` call in
         their `calculate_reward()`:
 
         def calculate_reward(self, action: int) -> float:

freqtrade/freqai/base_models/FreqaiMultiOutputClassifier.py

@@ -1,9 +1,8 @@
 import numpy as np
-from joblib import Parallel
 from sklearn.base import is_classifier
 from sklearn.multioutput import MultiOutputClassifier, _fit_estimator
-from sklearn.utils.fixes import delayed
 from sklearn.utils.multiclass import check_classification_targets
+from sklearn.utils.parallel import Parallel, delayed
 from sklearn.utils.validation import has_fit_parameter
 
 from freqtrade.exceptions import OperationalException

freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py

@@ -1,6 +1,5 @@
-from joblib import Parallel
 from sklearn.multioutput import MultiOutputRegressor, _fit_estimator
-from sklearn.utils.fixes import delayed
+from sklearn.utils.parallel import Parallel, delayed
 from sklearn.utils.validation import has_fit_parameter
 
 

freqtrade/freqai/data_drawer.py

@@ -12,7 +12,6 @@ import numpy as np
 import pandas as pd
 import psutil
 import rapidjson
-from joblib import dump, load
 from joblib.externals import cloudpickle
 from numpy.typing import NDArray
 from pandas import DataFrame
@@ -285,6 +284,10 @@ class FreqaiDataDrawer:
         new_pred["date_pred"] = dataframe["date"]
         hist_preds = self.historic_predictions[pair].copy()
 
+        # ensure both dataframes have the same date format so they can be merged
+        new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"])
+        hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"])
+
         # find the closest common date between new_pred and historic predictions
         # and cut off the new_pred dataframe at that date
         common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner")
@@ -295,9 +298,10 @@ class FreqaiDataDrawer:
                            "predictions. You likely left your FreqAI instance offline "
                            f"for more than {len(dataframe.index)} candles.")
 
-        df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys())
-        # remove last row because we will append that later in append_model_predictions()
-        df_concat = df_concat.iloc[:-1]
+        # reindex new_pred columns to match the historic predictions dataframe
+        new_pred_reindexed = new_pred.reindex(columns=hist_preds.columns)
+        df_concat = pd.concat([hist_preds, new_pred_reindexed], ignore_index=True)
+
         # any missing values will get zeroed out so users can see the exact
         # downtime in FreqUI
         df_concat = df_concat.fillna(0)
@@ -319,9 +323,9 @@ class FreqaiDataDrawer:
         index = self.historic_predictions[pair].index[-1:]
         columns = self.historic_predictions[pair].columns
 
-        nan_df = pd.DataFrame(np.nan, index=index, columns=columns)
+        zeros_df = pd.DataFrame(np.zeros((1, len(columns))), index=index, columns=columns)
         self.historic_predictions[pair] = pd.concat(
-            [self.historic_predictions[pair], nan_df], ignore_index=True, axis=0)
+            [self.historic_predictions[pair], zeros_df], ignore_index=True, axis=0)
         df = self.historic_predictions[pair]
 
         # model outputs and associated statistics
@@ -472,7 +476,8 @@ class FreqaiDataDrawer:
 
         # Save the trained model
         if self.model_type == 'joblib':
-            dump(model, save_path / f"{dk.model_filename}_model.joblib")
+            with (save_path / f"{dk.model_filename}_model.joblib").open("wb") as fp:
+                cloudpickle.dump(model, fp)
         elif self.model_type == 'keras':
             model.save(save_path / f"{dk.model_filename}_model.h5")
         elif self.model_type in ["stable_baselines3", "sb3_contrib", "pytorch"]:
@@ -559,7 +564,8 @@ class FreqaiDataDrawer:
         if dk.live and coin in self.model_dictionary:
             model = self.model_dictionary[coin]
         elif self.model_type == 'joblib':
-            model = load(dk.data_path / f"{dk.model_filename}_model.joblib")
+            with (dk.data_path / f"{dk.model_filename}_model.joblib").open("rb") as fp:
+                model = cloudpickle.load(fp)
         elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type:
             mod = importlib.import_module(
                 self.model_type, self.freqai_info['rl_config']['model_type'])

freqtrade/freqai/data_kitchen.py

@@ -244,7 +244,7 @@ class FreqaiDataKitchen:
                 f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points"
                 f" due to NaNs in populated dataset {len(unfiltered_df)}."
             )
-            if len(unfiltered_df) == 0 and not self.live:
+            if len(filtered_df) == 0 and not self.live:
                 raise OperationalException(
                     f"{self.pair}: all training data dropped due to NaNs. "
                     "You likely did not download enough training data prior "
@@ -255,7 +255,7 @@ class FreqaiDataKitchen:
             if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live:
                 worst_indicator = str(unfiltered_df.count().idxmin())
                 logger.warning(
-                    f" {(1 - len(filtered_df)/len(unfiltered_df)) * 100:.0f} percent "
+                    f" {(1 - len(filtered_df) / len(unfiltered_df)) * 100:.0f} percent "
                     " of training data dropped due to NaNs, model may perform inconsistent "
                     f"with expectations. Verify {worst_indicator}"
                 )
@@ -432,8 +432,12 @@ class FreqaiDataKitchen:
         if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0:
             append_df["DI_values"] = self.DI_values
 
+        user_cols = [col for col in dataframe_backtest.columns if col.startswith("%%")]
+        cols = ["date"]
+        cols.extend(user_cols)
+
         dataframe_backtest.reset_index(drop=True, inplace=True)
-        merged_df = pd.concat([dataframe_backtest["date"], append_df], axis=1)
+        merged_df = pd.concat([dataframe_backtest[cols], append_df], axis=1)
         return merged_df
 
     def append_predictions(self, append_df: DataFrame) -> None:
@@ -451,7 +455,8 @@ class FreqaiDataKitchen:
         Back fill values to before the backtesting range so that the dataframe matches size
         when it goes back to the strategy. These rows are not included in the backtest.
         """
-        to_keep = [col for col in dataframe.columns if not col.startswith("&")]
+        to_keep = [col for col in dataframe.columns if
+                   not col.startswith("&") and not col.startswith("%%")]
         self.return_dataframe = pd.merge(dataframe[to_keep],
                                          self.full_df, how='left', on='date')
         self.return_dataframe[self.full_df.columns] = (
@@ -709,6 +714,8 @@ class FreqaiDataKitchen:
                 pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs)
             informative_copy = informative_df.copy()
 
+            logger.debug(f"Populating features for {pair} {tf}")
+
             for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]:
                 df_features = strategy.feature_engineering_expand_all(
                     informative_copy.copy(), t, metadata=metadata)
@@ -788,6 +795,7 @@ class FreqaiDataKitchen:
 
         if not prediction_dataframe.empty:
             dataframe = prediction_dataframe.copy()
+            base_dataframes[self.config["timeframe"]] = dataframe.copy()
         else:
             dataframe = base_dataframes[self.config["timeframe"]].copy()
 

freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py

@@ -27,6 +27,12 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor):
         ...
         "freqai": {
             ...
+            "conv_width": 30,  // PyTorchTransformer is based on windowing
+            "feature_parameters": {
+                ...
+                "include_shifted_candles": 0,  // which removes the need for shifted candles
+                ...
+            },
             "model_training_parameters" : {
                 "learning_rate": 3e-4,
                 "trainer_kwargs": {
@@ -120,16 +126,16 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor):
         # create empty torch tensor
         self.model.model.eval()
         yb = torch.empty(0).to(self.device)
-        if x.shape[1] > 1:
+        if x.shape[1] > self.window_size:
             ws = self.window_size
             for i in range(0, x.shape[1] - ws):
                 xb = x[:, i:i + ws, :].to(self.device)
                 y = self.model.model(xb)
-                yb = torch.cat((yb, y), dim=0)
+                yb = torch.cat((yb, y), dim=1)
         else:
             yb = self.model.model(x)
 
-        yb = yb.cpu().squeeze()
+        yb = yb.cpu().squeeze(0)
         pred_df = pd.DataFrame(yb.detach().numpy(), columns=dk.label_list)
         pred_df, _, _ = dk.label_pipeline.inverse_transform(pred_df)
 

freqtrade/freqai/prediction_models/ReinforcementLearner.py

@@ -1,8 +1,9 @@
 import logging
 from pathlib import Path
-from typing import Any, Dict, Type
+from typing import Any, Dict, List, Optional, Type
 
 import torch as th
+from stable_baselines3.common.callbacks import ProgressBarCallback
 
 from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 from freqtrade.freqai.RL.Base5ActionRLEnv import Actions, Base5ActionRLEnv, Positions
@@ -73,19 +74,27 @@ class ReinforcementLearner(BaseReinforcementLearningModel):
                         'trained agent.')
             model = self.dd.model_dictionary[dk.pair]
             model.set_env(self.train_env)
+        callbacks: List[Any] = [self.eval_callback, self.tensorboard_callback]
+        progressbar_callback: Optional[ProgressBarCallback] = None
+        if self.rl_config.get('progress_bar', False):
+            progressbar_callback = ProgressBarCallback()
+            callbacks.insert(0, progressbar_callback)
 
-        model.learn(
-            total_timesteps=int(total_timesteps),
-            callback=[self.eval_callback, self.tensorboard_callback],
-            progress_bar=self.rl_config.get('progress_bar', False)
-        )
+        try:
+            model.learn(
+                total_timesteps=int(total_timesteps),
+                callback=callbacks,
+            )
+        finally:
+            if progressbar_callback:
+                progressbar_callback.on_training_end()
 
         if Path(dk.data_path / "best_model.zip").is_file():
             logger.info('Callback found a best model.')
             best_model = self.MODELCLASS.load(dk.data_path / "best_model")
             return best_model
 
-        logger.info('Couldnt find best model, using final model instead.')
+        logger.info("Couldn't find best model, using final model instead.")
 
         return model
 

freqtrade/freqai/prediction_models/SKLearnRandomForestClassifier.py

@@ -0,0 +1,82 @@
+import logging
+from typing import Any, Dict, Tuple
+
+import numpy as np
+import numpy.typing as npt
+from pandas import DataFrame
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+
+from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+
+
+logger = logging.getLogger(__name__)
+
+
+class SKLearnRandomForestClassifier(BaseClassifierModel):
+    """
+    User created prediction model. The class inherits IFreqaiModel, which
+    means it has full access to all Frequency AI functionality. Typically,
+    users would use this to override the common `fit()`, `train()`, or
+    `predict()` methods to add their custom data handling tools or change
+    various aspects of the training that cannot be configured via the
+    top level config.json file.
+    """
+
+    def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
+        """
+        User sets up the training and test data to fit their desired model here
+        :param data_dictionary: the dictionary holding all data for train, test,
+            labels, weights
+        :param dk: The datakitchen object for the current coin/model
+        """
+
+        X = data_dictionary["train_features"].to_numpy()
+        y = data_dictionary["train_labels"].to_numpy()[:, 0]
+
+        if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:
+            eval_set = None
+        else:
+            test_features = data_dictionary["test_features"].to_numpy()
+            test_labels = data_dictionary["test_labels"].to_numpy()[:, 0]
+
+            eval_set = (test_features, test_labels)
+
+        if self.freqai_info.get("continual_learning", False):
+            logger.warning("Continual learning is not supported for "
+                           "SKLearnRandomForestClassifier, ignoring.")
+
+        train_weights = data_dictionary["train_weights"]
+
+        model = RandomForestClassifier(**self.model_training_parameters)
+
+        model.fit(X=X, y=y, sample_weight=train_weights)
+        if eval_set:
+            logger.info("Score: %s", model.score(eval_set[0], eval_set[1]))
+
+        return model
+
+    def predict(
+        self, unfiltered_df: DataFrame, dk: FreqaiDataKitchen, **kwargs
+    ) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
+        """
+        Filter the prediction features data and predict with it.
+        :param  unfiltered_df: Full dataframe for the current backtest period.
+        :return:
+        :pred_df: dataframe containing the predictions
+        :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
+        data (NaNs) or felt uncertain about data (PCA and DI index)
+        """
+
+        (pred_df, dk.do_predict) = super().predict(unfiltered_df, dk, **kwargs)
+
+        le = LabelEncoder()
+        label = dk.label_list[0]
+        labels_before = list(dk.data['labels_std'].keys())
+        labels_after = le.fit_transform(labels_before).tolist()
+        pred_df[label] = le.inverse_transform(pred_df[label])
+        pred_df = pred_df.rename(
+            columns={labels_after[i]: labels_before[i] for i in range(len(labels_before))})
+
+        return (pred_df, dk.do_predict)

freqtrade/freqai/prediction_models/XGBoostRFRegressor.py

@@ -45,7 +45,7 @@ class XGBoostRFRegressor(BaseRegressionModel):
 
         model = XGBRFRegressor(**self.model_training_parameters)
 
-        model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard)
+        model.set_params(callbacks=[TBCallback(dk.data_path)])
         model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set,
                   sample_weight_eval_set=eval_weights, xgb_model=xgb_model)
         # set the callbacks to empty so that we can serialize to disk later

freqtrade/freqai/prediction_models/XGBoostRegressor.py

@@ -45,7 +45,7 @@ class XGBoostRegressor(BaseRegressionModel):
 
         model = XGBRegressor(**self.model_training_parameters)
 
-        model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard)
+        model.set_params(callbacks=[TBCallback(dk.data_path)])
         model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set,
                   sample_weight_eval_set=eval_weights, xgb_model=xgb_model)
         # set the callbacks to empty so that we can serialize to disk later

freqtrade/freqai/tensorboard/TensorboardCallback.py

@@ -3,7 +3,6 @@ from typing import Any, Dict, Type, Union
 
 from stable_baselines3.common.callbacks import BaseCallback
 from stable_baselines3.common.logger import HParam
-from stable_baselines3.common.vec_env import VecEnv
 
 from freqtrade.freqai.RL.BaseEnvironment import BaseActions
 
@@ -13,13 +12,9 @@ class TensorboardCallback(BaseCallback):
     Custom callback for plotting additional values in tensorboard and
     episodic summary reports.
     """
-    # Override training_env type to fix type errors
-    training_env: Union[VecEnv, None] = None
-
     def __init__(self, verbose=1, actions: Type[Enum] = BaseActions):
         super().__init__(verbose)
         self.model: Any = None
-        self.logger: Any = None
         self.actions: Type[Enum] = actions
 
     def _on_training_start(self) -> None:
@@ -47,9 +42,13 @@ class TensorboardCallback(BaseCallback):
     def _on_step(self) -> bool:
 
         local_info = self.locals["infos"][0]
-        if self.training_env is None:
-            return True
-        tensorboard_metrics = self.training_env.get_attr("tensorboard_metrics")[0]
+
+        if hasattr(self.training_env, 'envs'):
+            tensorboard_metrics = self.training_env.envs[0].unwrapped.tensorboard_metrics
+
+        else:
+            # For RL-multiproc - usage of [0] might need to be evaluated
+            tensorboard_metrics = self.training_env.get_attr("tensorboard_metrics")[0]
 
         for metric in local_info:
             if metric not in ["episode", "terminal_observation"]:

freqtrade/freqai/utils.py

@@ -13,7 +13,6 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
-from freqtrade.exchange.exchange import market_is_active
 from freqtrade.freqai.data_drawer import FreqaiDataDrawer
 from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
@@ -33,8 +32,11 @@ def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
 
     if dp._exchange is None:
         raise OperationalException('No exchange object found.')
-    markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
-               or config.get('include_inactive')]
+    markets = [
+        p for p in dp._exchange.get_markets(
+            tradable_only=True, active_only=not config.get('include_inactive')
+            ).keys()
+    ]
 
     all_pairs = dynamic_expand_pairlist(config, markets)
 
@@ -116,10 +118,12 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
         mdl = models[label]
         if "catboost.core" in str(mdl.__class__):
             feature_importance = mdl.get_feature_importance()
-        elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__):
+        elif "lightgbm.sklearn" in str(mdl.__class__):
+            feature_importance = mdl.feature_importances_
+        elif "xgb" in str(mdl.__class__):
             feature_importance = mdl.feature_importances_
         else:
-            logger.info('Model type not support for generating feature importances.')
+            logger.info('Model type does not support generating feature importances.')
             return
 
         # Data preparation

freqtrade/freqtradebot.py

@@ -18,8 +18,8 @@ from freqtrade.constants import BuySell, Config, EntryExecuteMode, ExchangeConfi
 from freqtrade.data.converter import order_book_to_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.edge import Edge
-from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, SignalDirection,
-                             State, TradingMode)
+from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, SignalDirection, State,
+                             TradingMode)
 from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, PricingError)
 from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, remove_exchange_credentials,
@@ -33,12 +33,12 @@ from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.rpc import RPCManager
 from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
-from freqtrade.rpc.rpc_types import (RPCBuyMsg, RPCCancelMsg, RPCProtectionMsg, RPCSellCancelMsg,
-                                     RPCSellMsg)
+from freqtrade.rpc.rpc_types import (ProfitLossStr, RPCCancelMsg, RPCEntryMsg, RPCExitCancelMsg,
+                                     RPCExitMsg, RPCProtectionMsg)
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.util import FtPrecise
-from freqtrade.util.binance_mig import migrate_binance_futures_names
+from freqtrade.util.migrations import migrate_binance_futures_names
 from freqtrade.wallets import Wallets
 
 
@@ -82,7 +82,8 @@ class FreqtradeBot(LoggingMixin):
 
         PairLocks.timeframe = self.config['timeframe']
 
-        self.pairlists = PairListManager(self.exchange, self.config)
+        self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
+        self.last_process: Optional[datetime] = None
 
         # RPC runs in separate threads, can start handling external commands just after
         # initialization, even before Freqtradebot has a chance to start its throttling,
@@ -119,8 +120,6 @@ class FreqtradeBot(LoggingMixin):
         self._exit_lock = Lock()
         LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe))
 
-        self.trading_mode: TradingMode = self.config.get('trading_mode', TradingMode.SPOT)
-
         self._schedule = Scheduler()
 
         if self.trading_mode == TradingMode.FUTURES:
@@ -129,13 +128,13 @@ class FreqtradeBot(LoggingMixin):
                 self.update_funding_fees()
                 self.wallets.update()
 
-            # TODO: This would be more efficient if scheduled in utc time, and performed at each
-            # TODO: funding interval, specified by funding_fee_times on the exchange classes
+            # This would be more efficient if scheduled in utc time, and performed at each
+            # funding interval, specified by funding_fee_times on the exchange classes
+            # However, this reduces the precision - and might therefore lead to problems.
             for time_slot in range(0, 24):
-                for minutes in [0, 15, 30, 45]:
+                for minutes in [1, 31]:
                     t = str(time(time_slot, minutes, 2))
                     self._schedule.every().day.at(t).do(update)
-        self.last_process: Optional[datetime] = None
 
         self.strategy.ft_bot_start()
         # Initialize protections AFTER bot start - otherwise parameters are not loaded.
@@ -199,6 +198,7 @@ class FreqtradeBot(LoggingMixin):
         # Only update open orders on startup
         # This will update the database after the initial migration
         self.startup_update_open_orders()
+        self.update_funding_fees()
 
     def process(self) -> None:
         """
@@ -312,22 +312,19 @@ class FreqtradeBot(LoggingMixin):
         open_trades = Trade.get_open_trade_count()
         return max(0, self.config['max_open_trades'] - open_trades)
 
-    def update_funding_fees(self):
+    def update_funding_fees(self) -> None:
         if self.trading_mode == TradingMode.FUTURES:
-            trades = Trade.get_open_trades()
-            try:
-                for trade in trades:
-                    funding_fees = self.exchange.get_funding_fees(
+            trades: List[Trade] = Trade.get_open_trades()
+            for trade in trades:
+                trade.set_funding_fees(
+                    self.exchange.get_funding_fees(
                         pair=trade.pair,
                         amount=trade.amount,
                         is_short=trade.is_short,
-                        open_date=trade.date_last_filled_utc
-                    )
-                    trade.funding_fees = funding_fees
-            except ExchangeError:
-                logger.warning("Could not update funding fees for open trades.")
+                        open_date=trade.date_last_filled_utc)
+                )
 
-    def startup_backpopulate_precision(self):
+    def startup_backpopulate_precision(self) -> None:
 
         trades = Trade.get_trades([Trade.contract_size.is_(None)])
         for trade in trades:
@@ -374,17 +371,13 @@ class FreqtradeBot(LoggingMixin):
                     fo = order.to_ccxt_object()
                     fo['status'] = 'canceled'
                     self.handle_cancel_order(
-                        fo, order.order_id, order.trade,
-                        constants.CANCEL_REASON['TIMEOUT']
+                        fo, order, order.trade, constants.CANCEL_REASON['TIMEOUT']
                     )
 
             except ExchangeError as e:
 
                 logger.warning(f"Error updating Order {order.order_id} due to {e}")
 
-        if self.trading_mode == TradingMode.FUTURES:
-            self._schedule.run_pending()
-
     def update_trades_without_assigned_fees(self) -> None:
         """
         Update closed trades without close fees assigned.
@@ -439,10 +432,6 @@ class FreqtradeBot(LoggingMixin):
             try:
                 fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
                                                                  order.ft_order_side == 'stoploss')
-                if order.ft_order_side == 'stoploss':
-                    if fo and fo['status'] == 'open':
-                        # Assume this as the open stoploss order
-                        trade.stoploss_order_id = order.order_id
                 if fo:
                     logger.info(f"Found {order} for trade {trade}.")
                     self.update_trade_state(trade, order.order_id, fo,
@@ -586,7 +575,8 @@ class FreqtradeBot(LoggingMixin):
                 else:
                     self.log_once(f"Pair {pair} is currently locked.", logger.info)
                 return False
-            stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
+            stake_amount = self.wallets.get_trade_stake_amount(
+                pair, self.config['max_open_trades'], self.edge)
 
             bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {})
             if ((bid_check_dom.get('enabled', False)) and
@@ -651,8 +641,7 @@ class FreqtradeBot(LoggingMixin):
         max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate)
         stake_available = self.wallets.get_available_stake_amount()
         logger.debug(f"Calling adjust_trade_position for pair {trade.pair}")
-        stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
-                                             default_retval=None, supress_error=True)(
+        stake_amount, order_tag = self.strategy._adjust_trade_position_internal(
             trade=trade,
             current_time=datetime.now(timezone.utc), current_rate=current_entry_rate,
             current_profit=current_entry_profit, min_stake=min_entry_stake,
@@ -671,33 +660,27 @@ class FreqtradeBot(LoggingMixin):
                 else:
                     logger.debug("Max adjustment entries is set to unlimited.")
             self.execute_entry(trade.pair, stake_amount, price=current_entry_rate,
-                               trade=trade, is_short=trade.is_short, mode='pos_adjust')
+                               trade=trade, is_short=trade.is_short, mode='pos_adjust',
+                               enter_tag=order_tag)
 
         if stake_amount is not None and stake_amount < 0.0:
             # We should decrease our position
             amount = self.exchange.amount_to_contract_precision(
                 trade.pair,
                 abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate))))
-            if amount > trade.amount:
-                # This is currently ineffective as remaining would become < min tradable
-                # Fixing this would require checking for 0.0 there -
-                # if we decide that this callback is allowed to "fully exit"
-                logger.info(
-                    f"Adjusting amount to trade.amount as it is higher. {amount} > {trade.amount}")
-                amount = trade.amount
 
             if amount == 0.0:
                 logger.info("Amount to exit is 0.0 due to exchange limits - not exiting.")
                 return
 
             remaining = (trade.amount - amount) * current_exit_rate
-            if min_exit_stake and remaining < min_exit_stake:
+            if min_exit_stake and remaining != 0 and remaining < min_exit_stake:
                 logger.info(f"Remaining amount of {remaining} would be smaller "
                             f"than the minimum of {min_exit_stake}.")
                 return
 
             self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple(
-                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount)
+                exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount, exit_tag=order_tag)
 
     def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool:
         """
@@ -719,7 +702,7 @@ class FreqtradeBot(LoggingMixin):
         delta = f"Delta: {bids_ask_delta}"
 
         logger.info(
-            f"{bids}, {asks}, {delta}, Direction: {side.value}"
+            f"{bids}, {asks}, {delta}, Direction: {side.value} "
             f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, "
             f"Immediate Bid Quantity: {order_book['bids'][0][1]}, "
             f"Immediate Ask Quantity: {order_book['asks'][0][1]}."
@@ -749,6 +732,7 @@ class FreqtradeBot(LoggingMixin):
         :param pair: pair for which we want to create a LIMIT_BUY
         :param stake_amount: amount of stake-currency for the pair
         :return: True if a buy order is created, false if it fails.
+        :raise: DependencyException or it's subclasses like ExchangeError.
         """
         time_in_force = self.strategy.order_time_in_force['entry']
 
@@ -794,6 +778,7 @@ class FreqtradeBot(LoggingMixin):
             leverage=leverage
         )
         order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested)
+        order_obj.ft_order_tag = enter_tag
         order_id = order['id']
         order_status = order.get('status')
         logger.info(f"Order {order_id} was created for {pair} and status is {order_status}.")
@@ -835,14 +820,15 @@ class FreqtradeBot(LoggingMixin):
         base_currency = self.exchange.get_pair_base_currency(pair)
         open_date = datetime.now(timezone.utc)
 
+        funding_fees = self.exchange.get_funding_fees(
+            pair=pair,
+            amount=amount + trade.amount if trade else amount,
+            is_short=is_short,
+            open_date=trade.date_last_filled_utc if trade else open_date
+        )
+
         # This is a new trade
         if trade is None:
-            funding_fees = 0.0
-            try:
-                funding_fees = self.exchange.get_funding_fees(
-                    pair=pair, amount=amount, is_short=is_short, open_date=open_date)
-            except ExchangeError:
-                logger.warning("Could not find funding fee.")
 
             trade = Trade(
                 pair=pair,
@@ -878,6 +864,7 @@ class FreqtradeBot(LoggingMixin):
             trade.is_open = True
             trade.fee_open_currency = None
             trade.open_rate_requested = enter_limit_requested
+            trade.set_funding_fees(funding_fees)
 
         trade.orders.append(order_obj)
         trade.recalc_trade_from_orders()
@@ -904,17 +891,15 @@ class FreqtradeBot(LoggingMixin):
 
     def cancel_stoploss_on_exchange(self, trade: Trade) -> Trade:
         # First cancelling stoploss on exchange ...
-        if trade.stoploss_order_id:
+        for oslo in trade.open_sl_orders:
             try:
-                logger.info(f"Canceling stoploss on exchange for {trade}")
+                logger.info(f"Cancelling stoploss on exchange for {trade} "
+                            f"order: {oslo.order_id}")
                 co = self.exchange.cancel_stoploss_order_with_result(
-                    trade.stoploss_order_id, trade.pair, trade.amount)
-                self.update_trade_state(trade, trade.stoploss_order_id, co, stoploss_order=True)
-
-                # Reset stoploss order id.
-                trade.stoploss_order_id = None
+                    oslo.order_id, trade.pair, trade.amount)
+                self.update_trade_state(trade, oslo.order_id, co, stoploss_order=True)
             except InvalidOrderException:
-                logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id} "
+                logger.exception(f"Could not cancel stoploss order {oslo.order_id} "
                                  f"for pair {trade.pair}")
         return trade
 
@@ -1002,7 +987,7 @@ class FreqtradeBot(LoggingMixin):
 
         return enter_limit_requested, stake_amount, leverage
 
-    def _notify_enter(self, trade: Trade, order: Order, order_type: str,
+    def _notify_enter(self, trade: Trade, order: Order, order_type: Optional[str],
                       fill: bool = False, sub_trade: bool = False) -> None:
         """
         Sends rpc notification when a entry order occurred.
@@ -1012,12 +997,10 @@ class FreqtradeBot(LoggingMixin):
         if open_rate is None:
             open_rate = trade.open_rate
 
-        current_rate = trade.open_rate_requested
-        if self.dataprovider.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
-            current_rate = self.exchange.get_rate(
-                trade.pair, side='entry', is_short=trade.is_short, refresh=False)
+        current_rate = self.exchange.get_rate(
+            trade.pair, side='entry', is_short=trade.is_short, refresh=False)
 
-        msg: RPCBuyMsg = {
+        msg: RPCEntryMsg = {
             'trade_id': trade.id,
             'type': RPCMessageType.ENTRY_FILL if fill else RPCMessageType.ENTRY,
             'buy_tag': trade.enter_tag,
@@ -1028,10 +1011,11 @@ class FreqtradeBot(LoggingMixin):
             'direction': 'Short' if trade.is_short else 'Long',
             'limit': open_rate,  # Deprecated (?)
             'open_rate': open_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'amount': order.safe_amount_after_fee if fill else (order.amount or trade.amount),
             'open_date': trade.open_date_utc or datetime.now(timezone.utc),
@@ -1065,6 +1049,7 @@ class FreqtradeBot(LoggingMixin):
             'open_rate': trade.open_rate,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'amount': trade.amount,
             'open_date': trade.open_date,
@@ -1087,7 +1072,11 @@ class FreqtradeBot(LoggingMixin):
         trades_closed = 0
         for trade in trades:
 
-            if not trade.has_open_orders and not self.wallets.check_exit_amount(trade):
+            if (
+                not trade.has_open_orders
+                and not trade.has_open_sl_orders
+                and not self.wallets.check_exit_amount(trade)
+            ):
                 logger.warning(
                     f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. '
                     'Trying to recover.')
@@ -1195,8 +1184,6 @@ class FreqtradeBot(LoggingMixin):
             order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss',
                                                      trade.amount, stop_price)
             trade.orders.append(order_obj)
-            trade.stoploss_order_id = str(stoploss_order['id'])
-            trade.stoploss_last_update = datetime.now(timezone.utc)
             return True
         except InsufficientFundsError as e:
             logger.warning(f"Unable to place stoploss order {e}.")
@@ -1204,13 +1191,11 @@ class FreqtradeBot(LoggingMixin):
             self.handle_insufficient_funds(trade)
 
         except InvalidOrderException as e:
-            trade.stoploss_order_id = None
             logger.error(f'Unable to place a stoploss order on exchange. {e}')
             logger.warning('Exiting the trade forcefully')
             self.emergency_exit(trade, stop_price)
 
         except ExchangeError:
-            trade.stoploss_order_id = None
             logger.exception('Unable to place a stoploss order on exchange.')
         return False
 
@@ -1224,27 +1209,28 @@ class FreqtradeBot(LoggingMixin):
         """
 
         logger.debug('Handling stoploss on exchange %s ...', trade)
-        stoploss_order = None
 
-        try:
-            # First we check if there is already a stoploss on exchange
-            stoploss_order = self.exchange.fetch_stoploss_order(
-                trade.stoploss_order_id, trade.pair) if trade.stoploss_order_id else None
-        except InvalidOrderException as exception:
-            logger.warning('Unable to fetch stoploss order: %s', exception)
+        stoploss_orders = []
+        for slo in trade.open_sl_orders:
+            stoploss_order = None
+            try:
+                # First we check if there is already a stoploss on exchange
+                stoploss_order = self.exchange.fetch_stoploss_order(
+                    slo.order_id, trade.pair) if slo.order_id else None
+            except InvalidOrderException as exception:
+                logger.warning('Unable to fetch stoploss order: %s', exception)
 
-        if stoploss_order:
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
+            if stoploss_order:
+                stoploss_orders.append(stoploss_order)
+                self.update_trade_state(trade, slo.order_id, stoploss_order,
+                                        stoploss_order=True)
 
-        # We check if stoploss order is fulfilled
-        if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
-            trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
-            self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
-                                    stoploss_order=True)
-            self._notify_exit(trade, "stoploss", True)
-            self.handle_protections(trade.pair, trade.trade_direction)
-            return True
+            # We check if stoploss order is fulfilled
+            if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
+                trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
+                self._notify_exit(trade, "stoploss", True)
+                self.handle_protections(trade.pair, trade.trade_direction)
+                return True
 
         if trade.has_open_orders or not trade.is_open:
             # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case
@@ -1253,7 +1239,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         # If enter order is fulfilled but there is no stoploss, we add a stoploss on exchange
-        if not stoploss_order:
+        if len(stoploss_orders) == 0:
             stop_price = trade.stoploss_or_liquidation
             if self.edge:
                 stoploss = self.edge.get_stoploss(pair=trade.pair)
@@ -1267,27 +1253,7 @@ class FreqtradeBot(LoggingMixin):
                 # in which case the trade will be closed - which we must check below.
                 return False
 
-        # If stoploss order is canceled for some reason we add it again
-        if (trade.is_open
-                and stoploss_order
-                and stoploss_order['status'] in ('canceled', 'cancelled')):
-            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
-                return False
-            else:
-                logger.warning('Stoploss order was cancelled, but unable to recreate one.')
-
-        # Finally we check if stoploss on exchange should be moved up because of trailing.
-        # Triggered Orders are now real orders - so don't replace stoploss anymore
-        if (
-            trade.is_open and stoploss_order
-            and stoploss_order.get('status_stop') != 'triggered'
-            and (self.config.get('trailing_stop', False)
-                 or self.config.get('use_custom_stoploss', False))
-        ):
-            # if trailing stoploss is enabled we check if stoploss value has changed
-            # in which case we cancel stoploss order and put another one with new
-            # value immediately
-            self.handle_trailing_stoploss_on_exchange(trade, stoploss_order)
+        self.manage_trade_stoploss_orders(trade, stoploss_orders)
 
         return False
 
@@ -1323,6 +1289,42 @@ class FreqtradeBot(LoggingMixin):
                     logger.warning(f"Could not create trailing stoploss order "
                                    f"for pair {trade.pair}.")
 
+    def manage_trade_stoploss_orders(self, trade: Trade, stoploss_orders: List[Dict]):
+        """
+        Perform required actions acording to existing stoploss orders of trade
+        :param trade: Corresponding Trade
+        :param stoploss_orders: Current on exchange stoploss orders
+        :return: None
+        """
+        # If all stoploss orderd are canceled for some reason we add it again
+        canceled_sl_orders = [o for o in stoploss_orders
+                              if o['status'] in ('canceled', 'cancelled')]
+        if (
+            trade.is_open and
+            len(stoploss_orders) > 0 and
+            len(stoploss_orders) == len(canceled_sl_orders)
+        ):
+            if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
+                return False
+            else:
+                logger.warning('All Stoploss orders are cancelled, but unable to recreate one.')
+
+        active_sl_orders = [o for o in stoploss_orders if o not in canceled_sl_orders]
+        if len(active_sl_orders) > 0:
+            last_active_sl_order = active_sl_orders[-1]
+            # Finally we check if stoploss on exchange should be moved up because of trailing.
+            # Triggered Orders are now real orders - so don't replace stoploss anymore
+            if (trade.is_open and
+                    last_active_sl_order.get('status_stop') != 'triggered' and
+                    (self.config.get('trailing_stop', False) or
+                     self.config.get('use_custom_stoploss', False))):
+                # if trailing stoploss is enabled we check if stoploss value has changed
+                # in which case we cancel stoploss order and put another one with new
+                # value immediately
+                self.handle_trailing_stoploss_on_exchange(trade, last_active_sl_order)
+
+        return
+
     def manage_open_orders(self) -> None:
         """
         Management of open orders on exchange. Unfilled orders might be cancelled if timeout
@@ -1331,6 +1333,7 @@ class FreqtradeBot(LoggingMixin):
         :return: None
         """
         for trade in Trade.get_open_trades():
+            open_order: Order
             for open_order in trade.open_orders:
                 try:
                     order = self.exchange.fetch_order(open_order.order_id, trade.pair)
@@ -1345,28 +1348,31 @@ class FreqtradeBot(LoggingMixin):
                 not_closed = order['status'] == 'open' or fully_cancelled
 
                 if not_closed:
-                    if fully_cancelled or (
-                        open_order and self.strategy.ft_check_timed_out(
-                            trade, open_order, datetime.now(timezone.utc)
+                    if (
+                        fully_cancelled or (
+                            open_order and self.strategy.ft_check_timed_out(
+                                trade, open_order, datetime.now(timezone.utc)
+                            )
                         )
                     ):
                         self.handle_cancel_order(
-                            order, open_order.order_id, trade, constants.CANCEL_REASON['TIMEOUT']
+                            order, open_order, trade, constants.CANCEL_REASON['TIMEOUT']
                         )
                     else:
                         self.replace_order(order, open_order, trade)
 
-    def handle_cancel_order(self, order: Dict, order_id: str, trade: Trade, reason: str) -> None:
+    def handle_cancel_order(self, order: Dict, order_obj: Order, trade: Trade, reason: str) -> None:
         """
         Check if current analyzed order timed out and cancel if necessary.
         :param order: Order dict grabbed with exchange.fetch_order()
+        :param order_obj: Order object from the database.
         :param trade: Trade object.
         :return: None
         """
         if order['side'] == trade.entry_side:
-            self.handle_cancel_enter(trade, order, order_id, reason)
+            self.handle_cancel_enter(trade, order, order_obj, reason)
         else:
-            canceled = self.handle_cancel_exit(trade, order, order_id, reason)
+            canceled = self.handle_cancel_exit(trade, order, order_obj, reason)
             canceled_count = trade.get_canceled_exit_order_count()
             max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0)
             if (canceled and max_timeouts > 0 and canceled_count >= max_timeouts):
@@ -1426,21 +1432,21 @@ class FreqtradeBot(LoggingMixin):
             # New candle
             proposed_rate = self.exchange.get_rate(
                 trade.pair, side='entry', is_short=trade.is_short, refresh=True)
-            adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price,
-                                                         default_retval=order_obj.price)(
+            adjusted_entry_price = strategy_safe_wrapper(
+                self.strategy.adjust_entry_price, default_retval=order_obj.safe_placement_price)(
                 trade=trade, order=order_obj, pair=trade.pair,
                 current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate,
-                current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag,
-                side=trade.entry_side)
+                current_order_rate=order_obj.safe_placement_price, entry_tag=trade.enter_tag,
+                side=trade.trade_direction)
 
             replacing = True
             cancel_reason = constants.CANCEL_REASON['REPLACE']
             if not adjusted_entry_price:
                 replacing = False
                 cancel_reason = constants.CANCEL_REASON['USER_CANCEL']
-            if order_obj.price != adjusted_entry_price:
+            if order_obj.safe_placement_price != adjusted_entry_price:
                 # cancel existing order if new price is supplied or None
-                res = self.handle_cancel_enter(trade, order, order_obj.order_id, cancel_reason,
+                res = self.handle_cancel_enter(trade, order, order_obj, cancel_reason,
                                                replacing=replacing)
                 if not res:
                     self.replace_order_failed(
@@ -1448,15 +1454,21 @@ class FreqtradeBot(LoggingMixin):
                     return
                 if adjusted_entry_price:
                     # place new order only if new price is supplied
-                    if not self.execute_entry(
-                        pair=trade.pair,
-                        stake_amount=(
-                            order_obj.safe_remaining * order_obj.safe_price / trade.leverage),
-                        price=adjusted_entry_price,
-                        trade=trade,
-                        is_short=trade.is_short,
-                        mode='replace',
-                    ):
+                    try:
+                        if not self.execute_entry(
+                            pair=trade.pair,
+                            stake_amount=(
+                                order_obj.safe_remaining * order_obj.safe_price / trade.leverage),
+                            price=adjusted_entry_price,
+                            trade=trade,
+                            is_short=trade.is_short,
+                            mode='replace',
+                        ):
+                            self.replace_order_failed(
+                                trade, f"Could not replace order for {trade}.")
+                    except DependencyException as exception:
+                        logger.warning(
+                            f'Unable to replace order for {trade.pair}: {exception}')
                         self.replace_order_failed(trade, f"Could not replace order for {trade}.")
 
     def cancel_all_open_orders(self) -> None:
@@ -1475,29 +1487,28 @@ class FreqtradeBot(LoggingMixin):
 
                 if order['side'] == trade.entry_side:
                     self.handle_cancel_enter(
-                        trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED']
+                        trade, order, open_order, constants.CANCEL_REASON['ALL_CANCELLED']
                     )
 
                 elif order['side'] == trade.exit_side:
                     self.handle_cancel_exit(
-                        trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED']
+                        trade, order, open_order, constants.CANCEL_REASON['ALL_CANCELLED']
                     )
         Trade.commit()
 
     def handle_cancel_enter(
-            self, trade: Trade, order: Dict, order_id: str,
+            self, trade: Trade, order: Dict, order_obj: Order,
             reason: str, replacing: Optional[bool] = False
     ) -> bool:
         """
         entry cancel - cancel order
+        :param order_obj: Order object from the database.
         :param replacing: Replacing order - prevent trade deletion.
         :return: True if trade was fully cancelled
         """
         was_trade_fully_canceled = False
+        order_id = order_obj.order_id
         side = trade.entry_side.capitalize()
-        if not trade.has_open_orders:
-            logger.warning(f"No open order for {trade}.")
-            return False
 
         if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
             filled_val: float = order.get('filled', 0.0) or 0.0
@@ -1510,8 +1521,8 @@ class FreqtradeBot(LoggingMixin):
                     f"Order {order_id} for {trade.pair} not cancelled, "
                     f"as the filled amount of {filled_val} would result in an unexitable trade.")
                 return False
-            corder = self.exchange.cancel_order_with_result(order_id, trade.pair,
-                                                            trade.amount)
+            corder = self.exchange.cancel_order_with_result(order_id, trade.pair, trade.amount)
+            order_obj.ft_cancel_reason = reason
             # if replacing, retry fetching the order 3 times if the status is not what we need
             if replacing:
                 retry_count = 0
@@ -1532,9 +1543,10 @@ class FreqtradeBot(LoggingMixin):
         else:
             # Order was cancelled already, so we can reuse the existing dict
             corder = order
-            reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
+            if order_obj.ft_cancel_reason is None:
+                order_obj.ft_cancel_reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
 
-        logger.info(f'{side} order {reason} for {trade}.')
+        logger.info(f'{side} order {order_obj.ft_cancel_reason} for {trade}.')
 
         # Using filled to determine the filled amount
         filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled')
@@ -1547,7 +1559,7 @@ class FreqtradeBot(LoggingMixin):
             if open_order_count < 1 and trade.nr_of_successful_entries == 0 and not replacing:
                 logger.info(f'{side} order fully cancelled. Removing {trade} from database.')
                 trade.delete()
-                reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
+                order_obj.ft_cancel_reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
             else:
                 self.update_trade_state(trade, order_id, corder)
                 logger.info(f'{side} Order timeout for {trade}.')
@@ -1557,21 +1569,21 @@ class FreqtradeBot(LoggingMixin):
             self.update_trade_state(trade, order_id, corder)
 
             logger.info(f'Partial {trade.entry_side} order timeout for {trade}.')
-            reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
+            order_obj.ft_cancel_reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
 
         self.wallets.update()
         self._notify_enter_cancel(trade, order_type=self.strategy.order_types['entry'],
-                                  reason=reason)
+                                  reason=order_obj.ft_cancel_reason)
         return was_trade_fully_canceled
 
     def handle_cancel_exit(
-        self, trade: Trade, order: Dict, order_id: str,
-        reason: str
+        self, trade: Trade, order: Dict, order_obj: Order, reason: str
     ) -> bool:
         """
         exit order cancel - cancel order and update trade
         :return: True if exit order was cancelled, false otherwise
         """
+        order_id = order_obj.order_id
         cancelled = False
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
@@ -1596,7 +1608,7 @@ class FreqtradeBot(LoggingMixin):
                         sub_trade=trade.amount != order['amount']
                     )
                     return False
-
+            order_obj.ft_cancel_reason = reason
             try:
                 order = self.exchange.cancel_order_with_result(
                     order['id'], trade.pair, trade.amount)
@@ -1615,19 +1627,22 @@ class FreqtradeBot(LoggingMixin):
                 trade.exit_reason = exit_reason_prev
             cancelled = True
         else:
-            reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
+            if order_obj.ft_cancel_reason is None:
+                order_obj.ft_cancel_reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
             trade.exit_reason = None
 
         self.update_trade_state(trade, order['id'], order)
 
-        logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
+        logger.info(
+            f'{trade.exit_side.capitalize()} order {order_obj.ft_cancel_reason} for {trade}.')
         trade.close_rate = None
         trade.close_rate_requested = None
 
         self._notify_exit_cancel(
             trade,
             order_type=self.strategy.order_types['exit'],
-            reason=reason, order_id=order['id'], sub_trade=trade.amount != order['amount']
+            reason=order_obj.ft_cancel_reason, order_id=order['id'],
+            sub_trade=trade.amount != order['amount']
         )
         return cancelled
 
@@ -1679,15 +1694,13 @@ class FreqtradeBot(LoggingMixin):
         :param exit_check: CheckTuple with signal and reason
         :return: True if it succeeds False
         """
-        try:
-            trade.funding_fees = self.exchange.get_funding_fees(
+        trade.set_funding_fees(
+            self.exchange.get_funding_fees(
                 pair=trade.pair,
                 amount=trade.amount,
                 is_short=trade.is_short,
-                open_date=trade.date_last_filled_utc,
-            )
-        except ExchangeError:
-            logger.warning("Could not update funding fee.")
+                open_date=trade.date_last_filled_utc)
+        )
 
         exit_type = 'exit'
         exit_reason = exit_tag or exit_check.exit_reason
@@ -1748,6 +1761,7 @@ class FreqtradeBot(LoggingMixin):
             return False
 
         order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit)
+        order_obj.ft_order_tag = exit_reason
         trade.orders.append(order_obj)
 
         trade.exit_order_status = ''
@@ -1762,7 +1776,7 @@ class FreqtradeBot(LoggingMixin):
 
         return True
 
-    def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False,
+    def _notify_exit(self, trade: Trade, order_type: Optional[str], fill: bool = False,
                      sub_trade: bool = False, order: Optional[Order] = None) -> None:
         """
         Sends rpc notification when a sell occurred.
@@ -1781,9 +1795,9 @@ class FreqtradeBot(LoggingMixin):
             order_rate = trade.safe_close_rate
             profit = trade.calculate_profit(rate=order_rate)
             amount = trade.amount
-        gain = "profit" if profit.profit_ratio > 0 else "loss"
+        gain: ProfitLossStr = "profit" if profit.profit_ratio > 0 else "loss"
 
-        msg: RPCSellMsg = {
+        msg: RPCExitMsg = {
             'type': (RPCMessageType.EXIT_FILL if fill
                      else RPCMessageType.EXIT),
             'trade_id': trade.id,
@@ -1794,25 +1808,27 @@ class FreqtradeBot(LoggingMixin):
             'gain': gain,
             'limit': order_rate,  # Deprecated
             'order_rate': order_rate,
-            'order_type': order_type,
+            'order_type': order_type or 'unknown',
             'amount': amount,
             'open_rate': trade.open_rate,
             'close_rate': order_rate,
             'current_rate': current_rate,
-            'profit_amount': profit.profit_abs if fill else profit.total_profit,
+            'profit_amount': profit.profit_abs,
             'profit_ratio': profit.profit_ratio,
             'buy_tag': trade.enter_tag,
             'enter_tag': trade.enter_tag,
-            'sell_reason': trade.exit_reason,  # Deprecated
             'exit_reason': trade.exit_reason,
             'open_date': trade.open_date_utc,
             'close_date': trade.close_date_utc or datetime.now(timezone.utc),
             'stake_amount': trade.stake_amount,
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency'),
             'sub_trade': sub_trade,
             'cumulative_profit': trade.realized_profit,
+            'final_profit_ratio': trade.close_profit if not trade.is_open else None,
+            'is_final_exit': trade.is_open is False,
         }
 
         # Send the message
@@ -1835,9 +1851,9 @@ class FreqtradeBot(LoggingMixin):
         profit = trade.calculate_profit(rate=profit_rate)
         current_rate = self.exchange.get_rate(
             trade.pair, side='exit', is_short=trade.is_short, refresh=False)
-        gain = "profit" if profit.profit_ratio > 0 else "loss"
+        gain: ProfitLossStr = "profit" if profit.profit_ratio > 0 else "loss"
 
-        msg: RPCSellCancelMsg = {
+        msg: RPCExitCancelMsg = {
             'type': RPCMessageType.EXIT_CANCEL,
             'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
@@ -1854,12 +1870,12 @@ class FreqtradeBot(LoggingMixin):
             'profit_ratio': profit.profit_ratio,
             'buy_tag': trade.enter_tag,
             'enter_tag': trade.enter_tag,
-            'sell_reason': trade.exit_reason,  # Deprecated
             'exit_reason': trade.exit_reason,
             'open_date': trade.open_date,
             'close_date': trade.close_date or datetime.now(timezone.utc),
             'stake_currency': self.config['stake_currency'],
             'base_currency': self.exchange.get_pair_base_currency(trade.pair),
+            'quote_currency': self.exchange.get_pair_quote_currency(trade.pair),
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'reason': reason,
             'sub_trade': sub_trade,
@@ -1910,7 +1926,7 @@ class FreqtradeBot(LoggingMixin):
 
         if self.exchange.check_order_canceled_empty(order):
             # Trade has been cancelled on exchange
-            # Handling of this will happen in check_handle_timedout.
+            # Handling of this will happen in handle_cancel_order.
             return True
 
         order_obj_or_none = trade.select_order_by_order_id(order_id)
@@ -1967,15 +1983,16 @@ class FreqtradeBot(LoggingMixin):
             self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool):
         """send "fill" notifications"""
 
-        sub_trade = not isclose(order.safe_amount_after_fee,
-                                trade.amount, abs_tol=constants.MATH_CLOSE_PREC)
         if order.ft_order_side == trade.exit_side:
             # Exit notification
             if send_msg and not stoploss_order and order.order_id not in trade.open_orders_ids:
-                self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order)
+                self._notify_exit(trade, order.order_type, fill=True,
+                                  sub_trade=trade.is_open, order=order)
             if not trade.is_open:
                 self.handle_protections(trade.pair, trade.trade_direction)
         elif send_msg and order.order_id not in trade.open_orders_ids and not stoploss_order:
+            sub_trade = not isclose(order.safe_amount_after_fee,
+                                    trade.amount, abs_tol=constants.MATH_CLOSE_PREC)
             # Enter fill
             self._notify_enter(trade, order, order.order_type, fill=True, sub_trade=sub_trade)
 

freqtrade/loggers/set_log_levels.py

@@ -8,15 +8,13 @@ logger = logging.getLogger(__name__)
 def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None:
     """
     Set the logging level for third party libraries
+    :param verbosity: Verbosity level. amount of `-v` passed to the command line
     :return: None
     """
-
-    logging.getLogger('requests').setLevel(
-        logging.INFO if verbosity <= 1 else logging.DEBUG
-    )
-    logging.getLogger("urllib3").setLevel(
-        logging.INFO if verbosity <= 1 else logging.DEBUG
-    )
+    for logger_name in ('requests', 'urllib3', 'httpcore'):
+        logging.getLogger(logger_name).setLevel(
+            logging.INFO if verbosity <= 1 else logging.DEBUG
+        )
     logging.getLogger('ccxt.base.exchange').setLevel(
         logging.INFO if verbosity <= 2 else logging.DEBUG
     )

freqtrade/misc.py

@@ -3,6 +3,7 @@ Various tool function for Freqtrade and scripts
 """
 import gzip
 import logging
+from io import StringIO
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Mapping, Optional, TextIO, Union
 from urllib.parse import urlparse
@@ -10,41 +11,12 @@ from urllib.parse import urlparse
 import pandas as pd
 import rapidjson
 
-from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
 from freqtrade.enums import SignalTagType, SignalType
 
 
 logger = logging.getLogger(__name__)
 
 
-def decimals_per_coin(coin: str):
-    """
-    Helper method getting decimal amount for this coin
-    example usage: f".{decimals_per_coin('USD')}f"
-    :param coin: Which coin are we printing the price / value for
-    """
-    return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK)
-
-
-def round_coin_value(
-        value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str:
-    """
-    Get price value for this coin
-    :param value: Value to be printed
-    :param coin: Which coin are we printing the price / value for
-    :param show_coin_name: Return string in format: "222.22 USDT" or "222.22"
-    :param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2"
-    :return: Formatted / rounded value (with or without coin name)
-    """
-    val = f"{value:.{decimals_per_coin(coin)}f}"
-    if not keep_trailing_zeros:
-        val = val.rstrip('0').rstrip('.')
-    if show_coin_name:
-        val = f"{val} {coin}"
-
-    return val
-
-
 def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None:
     """
     Dump JSON data into a file
@@ -231,7 +203,7 @@ def json_to_dataframe(data: str) -> pd.DataFrame:
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pd.read_json(data, orient='split')
+    dataframe = pd.read_json(StringIO(data), orient='split')
     if 'date' in dataframe.columns:
         dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
 

freqtrade/optimize/analysis/lookahead.py

@@ -94,8 +94,8 @@ class LookaheadAnalysis(BaseAnalysis):
                     # compare_df now comprises tuples with [1] having either 'self' or 'other'
                     if 'other' in col_name[1]:
                         continue
-                    self_value = compare_df_row[col_idx]
-                    other_value = compare_df_row[col_idx + 1]
+                    self_value = compare_df_row.iloc[col_idx]
+                    other_value = compare_df_row.iloc[col_idx + 1]
 
                     # output differences
                     if self_value != other_value:

freqtrade/optimize/analysis/lookahead_helpers.py

@@ -7,7 +7,7 @@ import pandas as pd
 
 from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
-from freqtrade.optimize.lookahead_analysis import LookaheadAnalysis
+from freqtrade.optimize.analysis.lookahead import LookaheadAnalysis
 from freqtrade.resolvers import StrategyResolver
 
 

freqtrade/optimize/analysis/recursive.py

@@ -64,7 +64,7 @@ class RecursiveAnalysis(BaseAnalysis):
                         self.dict_recursive[indicator][part.startup_candle] = f"{diff:.3f}%"
 
             else:
-                logger.info("No difference found. Stop the process.")
+                logger.info("No variance on indicator(s) found due to recursive formula.")
                 break
 
     # For lookahead bias check
@@ -100,7 +100,7 @@ class RecursiveAnalysis(BaseAnalysis):
                     # logger.info("part value {:.5f}".format(values_diff_other))
 
         else:
-            logger.info("No lookahead bias on indicators found. Stop the process.")
+            logger.info("No lookahead bias on indicators found.")
 
     def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]):
 
@@ -120,6 +120,7 @@ class RecursiveAnalysis(BaseAnalysis):
         prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load
 
         backtesting = Backtesting(prepare_data_config, self.exchange)
+        self.exchange = backtesting.exchange
         backtesting._set_strategy(backtesting.strategylist[0])
 
         varholder.data, varholder.timerange = backtesting.load_bt_data()

freqtrade/optimize/analysis/recursive_helpers.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, List
 
 from freqtrade.constants import Config
 from freqtrade.exceptions import OperationalException
-from freqtrade.optimize.recursive_analysis import RecursiveAnalysis
+from freqtrade.optimize.analysis.recursive import RecursiveAnalysis
 from freqtrade.resolvers import StrategyResolver
 
 
@@ -31,10 +31,13 @@ class RecursiveAnalysisSubFunctions:
                         temp_data.append(values.get(int(candle), '-'))
                     data.append(temp_data)
 
-        from tabulate import tabulate
-        table = tabulate(data, headers=headers, tablefmt="orgtbl")
-        print(table)
-        return table, headers, data
+        if len(data) > 0:
+            from tabulate import tabulate
+            table = tabulate(data, headers=headers, tablefmt="orgtbl")
+            print(table)
+            return table, headers, data
+
+        return None, None, data
 
     @staticmethod
     def calculate_config_overrides(config: Config):
@@ -81,8 +84,7 @@ class RecursiveAnalysisSubFunctions:
         if not (strategy_list := config.get('strategy_list', [])):
             if config.get('strategy') is None:
                 raise OperationalException(
-                    "No Strategy specified. Please specify a strategy via --strategy or "
-                    "--strategy-list"
+                    "No Strategy specified. Please specify a strategy via --strategy"
                 )
             strategy_list = [config['strategy']]
 
@@ -100,7 +102,5 @@ class RecursiveAnalysisSubFunctions:
             RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances(
                 RecursiveAnalysis_instances)
         else:
-            logger.error("There were no strategies specified neither through "
-                         "--strategy nor through "
-                         "--strategy-list "
+            logger.error("There was no strategy specified through --strategy "
                          "or timeframe was not specified.")