Merge pull request #9242 from freqtrade/new_release

New release 2023.9

.github/dependabot.yml

@@ -10,7 +10,7 @@ updates:
   directory: "/"
   schedule:
     interval: weekly
-  open-pull-requests-limit: 10
+  open-pull-requests-limit: 15
   target-branch: develop
 
 - package-ecosystem: "github-actions"

.github/workflows/ci.yml

@@ -25,10 +25,10 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -127,10 +127,10 @@ jobs:
     strategy:
       matrix:
         os: [ macos-latest ]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -237,10 +237,10 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -304,7 +304,7 @@ jobs:
   mypy_version_check:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -319,7 +319,7 @@ jobs:
   pre-commit:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-python@v4
       with:
@@ -329,7 +329,7 @@ jobs:
   docs_check:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Documentation syntax
       run: |
@@ -359,7 +359,7 @@ jobs:
     # Run pytest with "live" checks
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -443,12 +443,12 @@ jobs:
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.9"
+        python-version: "3.11"
 
     - name: Extract branch name
       shell: bash
@@ -461,7 +461,7 @@ jobs:
         python setup.py sdist bdist_wheel
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.8.8
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       if: (github.event_name == 'release')
       with:
         user: __token__
@@ -469,7 +469,7 @@ jobs:
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.8
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       if: (github.event_name == 'release')
       with:
         user: __token__
@@ -515,7 +515,7 @@ jobs:
     if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade'
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Extract branch name
       shell: bash

.github/workflows/docker_update_readme.yml

@@ -8,7 +8,7 @@ jobs:
   dockerHubDescription:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Docker Hub Description
       uses: peter-evans/dockerhub-description@v3
       env:

.gitignore

@@ -83,6 +83,9 @@ instance/
 # Scrapy stuff:
 .scrapy
 
+# memray
+memray-*
+
 # Sphinx documentation
 docs/_build/
 # Mkdocs documentation

.pre-commit-config.yaml

@@ -8,17 +8,17 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.3.0"
+    rev: "v1.5.1"
     hooks:
       - id: mypy
         exclude: build_helpers
         additional_dependencies:
           - types-cachetools==5.3.0.6
           - types-filelock==3.2.7
-          - types-requests==2.31.0.2
+          - types-requests==2.31.0.4
           - types-tabulate==0.9.0.3
           - types-python-dateutil==2.8.19.14
-          - SQLAlchemy==2.0.19
+          - SQLAlchemy==2.0.21
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

.readthedocs.yml

@@ -1,8 +1,14 @@
 # .readthedocs.yml
+version: 2
 
 build:
-  image: latest
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
 
 python:
-  version: 3.8
-  setup_py_install: false
+  install:
+    - requirements: docs/requirements-docs.txt
+
+mkdocs:
+  configuration: mkdocs.yml

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.4-slim-bullseye as base
+FROM python:3.11.5-slim-bullseye as base
 
 # Setup env
 ENV LANG C.UTF-8

README.md

@@ -59,7 +59,7 @@ Please find the complete documentation on the [freqtrade website](https://www.fr
 
 ## Features
 
-- [x] **Based on Python 3.8+**: For botting on any operating system - Windows, macOS and Linux.
+- [x] **Based on Python 3.9+**: For botting on any operating system - Windows, macOS and Linux.
 - [x] **Persistence**: Persistence is achieved through sqlite.
 - [x] **Dry-run**: Run the bot without paying money.
 - [x] **Backtesting**: Run a simulation of your buy/sell strategy.
@@ -207,7 +207,7 @@ To run this bot we recommend you a cloud instance with a minimum of:
 
 ### Software requirements
 
-- [Python >= 3.8](http://docs.python-guide.org/en/latest/starting/installation/)
+- [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/)
 - [pip](https://pip.pypa.io/en/stable/installing/)
 - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
 - [TA-Lib](https://ta-lib.github.io/ta-lib-python/)

build_helpers/install_ta-lib.sh

@@ -8,8 +8,9 @@ if [ -n "$2" ] || [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   tar zxvf ta-lib-0.4.0-src.tar.gz
   cd ta-lib \
   && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
-  && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.guess' -o config.guess \
-  && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.sub' -o config.sub \
+  && echo "Downloading gcc config.guess and config.sub" \
+  && curl -s 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.guess' -o config.guess \
+  && curl -s 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.sub' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
   && make
   if [ $? -ne 0 ]; then

build_helpers/install_windows.ps1

@@ -5,7 +5,7 @@ python -m pip install --upgrade pip wheel
 $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
 
 
-pip install --find-links=build_helpers\ TA-Lib
+pip install --find-links=build_helpers\ --prefer-binary  TA-Lib
 
 pip install -r requirements-dev.txt
 pip install -e .

config_examples/config_bittrex.example.json

@@ -32,11 +32,8 @@
         "name": "bittrex",
         "key": "your_exchange_key",
         "secret": "your_exchange_secret",
-        "ccxt_config": {"enableRateLimit": true},
-        "ccxt_async_config": {
-            "enableRateLimit": true,
-            "rateLimit": 500
-        },
+        "ccxt_config": {},
+        "ccxt_async_config": {},
         "pair_whitelist": [
             "ETH/BTC",
             "LTC/BTC",

config_examples/config_full.example.json

@@ -70,6 +70,7 @@
     },
     "pairlists": [
         {"method": "StaticPairList"},
+        {"method": "FullTradesFilter"},
         {
             "method": "VolumePairList",
             "number_assets": 20,
@@ -89,7 +90,6 @@
     ],
     "exchange": {
         "name": "binance",
-        "sandbox": false,
         "key": "your_exchange_key",
         "secret": "your_exchange_secret",
         "password": "",
@@ -206,6 +206,6 @@
     "recursive_strategy_search": false,
     "add_config_files": [],
     "reduce_df_footprint": false,
-    "dataformat_ohlcv": "json",
-    "dataformat_trades": "jsongz"
+    "dataformat_ohlcv": "feather",
+    "dataformat_trades": "feather"
 }

docker/Dockerfile.armhf

@@ -36,8 +36,9 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 # Install dependencies
 COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
 USER ftuser
-RUN  pip install --user --no-cache-dir numpy \
+RUN  pip install --user --no-cache-dir numpy==1.25.2 \
   && pip install --user /tmp/pyarrow-*.whl \
+  && pip install --user --no-build-isolation TA-Lib==0.4.28 \
   && pip install --user --no-cache-dir -r requirements.txt
 
 # Copy dependencies to runtime-image

docs/bot-basics.md

@@ -7,7 +7,7 @@ This page provides you some basic concepts on how Freqtrade works and operates.
 * **Strategy**: Your trading strategy, telling the bot what to do.
 * **Trade**: Open position.
 * **Open Order**: Order which is currently placed on the exchange, and is not yet complete.
-* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. XRP/USDT).
+* **Pair**: Tradable pair, usually in the format of Base/Quote (e.g. `XRP/USDT` for spot, `XRP/USDT:USDT` for futures).
 * **Timeframe**: Candle length to use (e.g. `"5m"`, `"1h"`, ...).
 * **Indicators**: Technical indicators (SMA, EMA, RSI, ...).
 * **Limit order**: Limit orders which execute at the defined limit price or better.
@@ -20,6 +20,20 @@ This page provides you some basic concepts on how Freqtrade works and operates.
 
 All profit calculations of Freqtrade include fees. For Backtesting / Hyperopt / Dry-run modes, the exchange default fee is used (lowest tier on the exchange). For live operations, fees are used as applied by the exchange (this includes BNB rebates etc.).
 
+## Pair naming
+
+Freqtrade follows the [ccxt naming convention](https://docs.ccxt.com/#/README?id=consistency-of-base-and-quote-currencies) for currencies.
+Using the wrong naming convention in the wrong market will usually result in the bot not recognizing the pair, usually resulting in errors like "this pair is not available".
+
+### Spot pair naming
+
+For spot pairs, naming will be `base/quote` (e.g. `ETH/USDT`).
+
+### Futures pair naming
+
+For futures pairs, naming will be `base/quote:settle` (e.g. `ETH/USDT:USDT`).
+
+
 ## Bot execution logic
 
 Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop.

docs/bot-usage.md

@@ -3,7 +3,7 @@
 This page explains the different parameters of the bot and how to run it.
 
 !!! Note
-    If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands.
+    If you've used `setup.sh`, don't forget to activate your virtual environment (`source .venv/bin/activate`) before running freqtrade commands.
 
 !!! Warning "Up-to-date clock"
     The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges.

docs/configuration.md

@@ -177,7 +177,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exit_pricing.order_book_top` | Bot will use the top N rate in Order Book "price_side" to exit. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Exit](#exit-price-with-orderbook-enabled)<br>*Defaults to `1`.* <br> **Datatype:** Positive Integer
 | `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price. <br>*Defaults to `0.02` 2%).*<br> **Datatype:** Positive float
 | | **TODO**
-| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
+| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. <br>Setting this to false disables the usage of `"exit_long"` and `"exit_short"` columns. Has no influence on other exit methods (Stoploss, ROI, callbacks). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `true`.* <br> **Datatype:** Boolean
 | `exit_profit_only` | Wait until the bot reaches `exit_profit_offset` before taking an exit decision. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
 | `exit_profit_offset` | Exit-signal is only active above this value. Only active in combination with `exit_profit_only=True`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0`.* <br> **Datatype:** Float (as ratio)
 | `ignore_roi_if_entry_signal` | Do not exit if the entry signal is still active. This setting takes preference over `minimal_roi` and `use_exit_signal`. [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
@@ -188,7 +188,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position). <br> [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `-1`.*<br> **Datatype:** Positive Integer or -1
 | | **Exchange**
 | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). <br> **Datatype:** String
-| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.<br> **Datatype:** Boolean
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
@@ -251,8 +250,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances. <br> **Datatype:** String, SQLAlchemy connect string
 | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file. <br> **Datatype:** String
 | `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.<br> *Defaults to `[]`*. <br> **Datatype:** List of strings
-| `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data. <br> *Defaults to `json`*. <br> **Datatype:** String
-| `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `jsongz`*. <br> **Datatype:** String
+| `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data. <br> *Defaults to `feather`*. <br> **Datatype:** String
+| `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `feather`*. <br> **Datatype:** String
 | `reduce_df_footprint` | Recast all numeric columns to float32/int32, with the objective of reducing ram/disk usage (and decreasing train/inference timing in FreqAI). (Currently only affects FreqAI use-cases) <br> **Datatype:** Boolean. <br> Default: `False`.
 
 ### Parameters in the strategy
@@ -614,6 +613,7 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo
 * Orders are simulated, and will not be posted to the exchange.
 * Market orders fill based on orderbook volume the moment the order is placed.
 * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings.
+* Limit orders will be converted to market orders if they cross the price by more than 1%.
 * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled.
 * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline.
 

docs/data-analysis.md

@@ -10,7 +10,7 @@ You can run this server using the following command: `docker compose -f docker/d
 This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
 Please use the link that's printed in the console after startup for simplified login.
 
-For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section.
+For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analysis-using-docker-compose) section.
 
 ### Pro tips
 
@@ -27,7 +27,7 @@ For this to work, first activate your virtual environment and run the following
 
 ``` bash
 # Activate virtual environment
-source .env/bin/activate
+source .venv/bin/activate
 
 pip install ipykernel
 ipython kernel install --user --name=freqtrade

docs/data-download.md

@@ -27,11 +27,11 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--exchange EXCHANGE]
                                [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
-                               [--data-format-trades {json,jsongz,hdf5}]
+                               [--data-format-trades {json,jsongz,hdf5,feather}]
                                [--trading-mode {spot,margin,futures}]
                                [--prepend]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
@@ -48,8 +48,7 @@ optional arguments:
   --dl-trades           Download trades instead of OHLCV data. The bot will
                         resample trades to the desired timeframe as specified
                         as --timeframes/-t.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
@@ -57,17 +56,18 @@ optional arguments:
                         exchange/pairs/timeframes.
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
-  --data-format-trades {json,jsongz,hdf5}
+                        (default: `feather`).
+  --data-format-trades {json,jsongz,hdf5,feather}
                         Storage format for downloaded trades data. (default:
-                        `jsongz`).
+                        `feather`).
   --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
                         Select Trading mode
   --prepend             Allow data prepending. (Data-appending is disabled)
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
@@ -154,13 +154,13 @@ freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --
 
 Freqtrade currently supports the following data-formats:
 
+* `feather` - a dataformat based on Apache Arrow
 * `json` -  plain "text" json files
 * `jsongz` - a gzip-zipped version of json files
 * `hdf5` - a high performance datastore
-* `feather` - a dataformat based on Apache Arrow (OHLCV only)
 * `parquet` - columnar datastore (OHLCV only)
 
-By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
+By default, both OHLCV data and trades data are stored in the `feather` format.
 
 This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively.
 To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
@@ -203,15 +203,15 @@ time freqtrade list-data --show-timerange --data-format-ohlcv <dataformat>
 
 |  Format | Size | timing |
 |------------|-------------|-------------|
+| `feather` | 72Mb | 3.5s |
 | `json` | 149Mb | 25.6s |
 | `jsongz` | 39Mb | 27s |
 | `hdf5` | 145Mb | 3.9s |
-| `feather` | 72Mb | 3.5s |
 | `parquet` | 83Mb | 3.8s |
 
 Size has been taken from the BTC/USDT 1m spot combination for the timerange specified above.
 
-To have a best performance/size mix, we recommend the use of either feather or parquet.
+To have a best performance/size mix, we recommend using the default feather format, or parquet.
 
 ### Pairs file
 
@@ -255,7 +255,7 @@ usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                               [--trading-mode {spot,margin,futures}]
                               [--candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...]]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
@@ -266,19 +266,20 @@ optional arguments:
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
   --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
                         Select Trading mode
   --candle-types {spot,futures,mark,index,premiumIndex,funding_rate} [{spot,futures,mark,index,premiumIndex,funding_rate} ...]
-                        Select candle type to use
+                        Select candle type to convert. Defaults to all
+                        available types.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
@@ -291,7 +292,6 @@ Common arguments:
                         Path to directory with historical backtesting data.
   --userdir PATH, --user-data-dir PATH
                         Path to userdata directory.
-
 ```
 
 ### Example converting data
@@ -314,7 +314,7 @@ usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                     {json,jsongz,hdf5,feather,parquet}
                                     [--erase] [--exchange EXCHANGE]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
@@ -325,12 +325,12 @@ optional arguments:
                         Destination format for data conversion.
   --erase               Clean all existing data for the selected
                         exchange/pairs/timeframes.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
@@ -367,9 +367,9 @@ usage: freqtrade trades-to-ohlcv [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                  [-t TIMEFRAMES [TIMEFRAMES ...]]
                                  [--exchange EXCHANGE]
                                  [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
-                                 [--data-format-trades {json,jsongz,hdf5}]
+                                 [--data-format-trades {json,jsongz,hdf5,feather}]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
@@ -377,18 +377,18 @@ optional arguments:
   -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
                         list. Default: `1m 5m`.
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
-  --data-format-trades {json,jsongz,hdf5}
+                        (default: `feather`).
+  --data-format-trades {json,jsongz,hdf5,feather}
                         Storage format for downloaded trades data. (default:
-                        `jsongz`).
+                        `feather`).
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
@@ -422,13 +422,12 @@ usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                            [--trading-mode {spot,margin,futures}]
                            [--show-timerange]
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
-  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
-                        config is provided.
+  --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
+                        (default: `feather`).
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
                         Limit command to these pairs. Pairs are space-
                         separated.
@@ -439,7 +438,8 @@ optional arguments:
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
-  --logfile FILE        Log to the file specified. Special values are:
+  --logfile FILE, --log-file FILE
+                        Log to the file specified. Special values are:
                         'syslog', 'journald'. See the documentation for more
                         details.
   -V, --version         show program's version number and exit
@@ -474,7 +474,7 @@ ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
 By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
 This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
 
-Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `<pair>-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+Since this data is large by default, the files use the feather fileformat by default. They are stored in your data-directory with the naming convention of `<pair>-trades.feather` (`ETH_BTC-trades.feather`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
 
 To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
 

docs/developer.md

@@ -77,7 +77,7 @@ def test_method_to_test(caplog):
 
 ### Debug configuration
 
-To debug freqtrade, we recommend VSCode with the following launch configuration (located in `.vscode/launch.json`).
+To debug freqtrade, we recommend VSCode (with the Python extension) with the following launch configuration (located in `.vscode/launch.json`).
 Details will obviously vary between setups - but this should work to get you started.
 
 ``` json
@@ -102,6 +102,19 @@ This method can also be used to debug a strategy, by setting the breakpoints wit
 
 A similar setup can also be taken for Pycharm - using `freqtrade` as module name, and setting the command line arguments as "parameters".
 
+??? Tip "Correct venv usage"
+    When using a virtual environment (which you should), make sure that your Editor is using the correct virtual environment to avoid problems or "unknown import" errors.
+
+    #### Vscode
+
+    You can select the correct environment in VSCode with the command "Python: Select Interpreter" - which will show you environments the extension detected.
+    If your environment has not been detected, you can also pick a path manually.
+
+    #### Pycharm
+
+    In pycharm, you can select the appropriate Environment in the "Run/Debug Configurations" window.
+    ![Pycharm debug configuration](assets/pycharm_debug.png)
+
 !!! Note "Startup directory"
     This assumes that you have the repository checked out, and the editor is started at the repository root level (so setup.py is at the top level of your repository).
 

docs/docker_quickstart.md

@@ -14,6 +14,9 @@ Start by downloading and installing Docker / Docker Desktop for your platform:
     Freqtrade documentation assumes the use of Docker desktop (or the docker compose plugin).  
     While the docker-compose standalone installation still works, it will require changing all `docker compose` commands from `docker compose` to `docker-compose` to work (e.g. `docker compose up -d` will become `docker-compose up -d`).
 
+??? Warning "Docker on windows"
+    If you just installed docker on a windows system, make sure to reboot your system, otherwise you might encounter unexplainable Problems related to network connectivity to docker containers.
+
 ## Freqtrade with docker
 
 Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage.
@@ -78,7 +81,7 @@ If you've selected to enable FreqUI in the `new-config` step, you will have freq
 
 You can now access the UI by typing localhost:8080 in your browser.
 
-??? Note "UI Access on a remote servers"
+??? Note "UI Access on a remote server"
     If you're running on a VPS, you should consider using either a ssh tunnel, or setup a VPN (openVPN, wireguard) to connect to your bot.
     This will ensure that freqUI is not directly exposed to the internet, which is not recommended for security reasons (freqUI does not support https out of the box).
     Setup of these tools is not part of this tutorial, however many good tutorials can be found on the internet.
@@ -128,7 +131,7 @@ All freqtrade arguments will be available by running `docker compose run --rm fr
 !!! Note "`docker compose run --rm`"
     Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command).
 
-??? Note "Using docker without docker"
+??? Note "Using docker without docker compose"
     "`docker compose run --rm`" will require a compose file to be provided.
     Some freqtrade commands that don't require authentication such as `list-pairs` can be run with "`docker run --rm`" instead.  
     For example `docker run --rm freqtradeorg/freqtrade:stable list-pairs --exchange binance --quote BTC --print-json`.  
@@ -172,7 +175,7 @@ You can then run `docker compose build --pull` to build the docker image, and ru
 
 ### Plotting with docker
 
-Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file.
+Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your `docker-compose.yml` file.
 You can then use these commands as follows:
 
 ``` bash
@@ -203,16 +206,20 @@ docker compose -f docker/docker-compose-jupyter.yml build --no-cache
 
 ### Docker on Windows
 
-* Error: `"Timestamp for this request is outside of the recvWindow."`
-  * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past.
-    To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so).
-    A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler.
+* Error: `"Timestamp for this request is outside of the recvWindow."`  
+  The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past.
+  To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so).
+  A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler.
 
-    ``` bash
-    taskkill /IM "Docker Desktop.exe" /F
-    wsl --shutdown
-    start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe"
-    ```
+  ``` bash
+  taskkill /IM "Docker Desktop.exe" /F
+  wsl --shutdown
+  start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe"
+  ```
+
+* Cannot connect to the API (Windows)  
+  If you're on windows and just installed Docker (desktop), make sure to reboot your System. Docker can have problems with network connectivity without a restart.
+  You should obviously also make sure to have your [settings](#accessing-the-ui) accordingly.
 
 !!! Warning
     Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting.

docs/edge.md

@@ -2,6 +2,10 @@
 
 The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.
 
+!!! Danger "Deprecated functionality"
+    `Edge positioning` (or short Edge) is currently in maintenance mode only (we keep existing functionality alive) and should be considered as deprecated.
+    It will currently not receive new features until either someone stepped forward to take up ownership of that module - or we'll decide to remove edge from freqtrade.
+
 !!! Warning
     When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
 

docs/exchanges.md

@@ -55,7 +55,7 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t
 ## Binance
 
 !!! Warning "Server location and geo-ip restrictions"
-    Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list.
+    Please be aware that Binance restricts API access regarding the server country. The current and non-exhaustive countries blocked are Canada, Malaysia, Netherlands and United States. Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list.
 
 Binance supports [time_in_force](configuration.md#understand-order_time_in_force).
 
@@ -136,15 +136,6 @@ Freqtrade will not attempt to change these settings.
 The Kraken API does only provide 720 historic candles, which is sufficient for Freqtrade dry-run and live trade modes, but is a problem for backtesting.
 To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data.
 
-Due to the heavy rate-limiting applied by Kraken, the following configuration section should be used to download data:
-
-``` json
-    "ccxt_async_config": {
-        "enableRateLimit": true,
-        "rateLimit": 3100
-    },
-```
-
 !!! Warning "Downloading data from kraken"
     Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine.
     It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient.

docs/faq.md

@@ -20,7 +20,7 @@ Futures trading is supported for selected exchanges. Please refer to the [docume
 
 * When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup).
 
-## Freqtrade common issues
+## Freqtrade common questions
 
 ### Can freqtrade open multiple positions on the same pair in parallel?
 
@@ -36,7 +36,7 @@ Running the bot with `freqtrade trade --config config.json` shows the output `fr
 This could be caused by the following reasons:
 
 * The virtual environment is not active.
-  * Run `source .env/bin/activate` to activate the virtual environment.
+  * Run `source .venv/bin/activate` to activate the virtual environment.
 * The installation did not complete successfully.
   * Please check the [Installation documentation](installation.md).
 
@@ -78,6 +78,14 @@ Where possible (e.g. on binance), the use of the exchange's dedicated fee curren
 On binance, it's sufficient to have BNB in your account, and have "Pay fees in BNB" enabled in your profile. Your BNB balance will slowly decline (as it's used to pay fees) - but you'll no longer encounter dust (Freqtrade will include the fees in the profit calculations).
 Other exchanges don't offer such possibilities, where it's simply something you'll have to accept or move to a different exchange.
 
+### I deposited more funds to the exchange, but my bot doesn't recognize this
+
+Freqtrade will update the exchange balance when necessary (Before placing an order).
+RPC calls (Telegram's `/balance`, API calls to `/balance`) can trigger an update at max. once per hour.
+
+If `adjust_trade_position` is enabled (and the bot has open trades eligible for position adjustments) - then the wallets will be refreshed once per hour.
+To force an immediate update, you can use `/reload_config` - which will restart the bot.
+
 ### I want to use incomplete candles
 
 Freqtrade will not provide incomplete candles to strategies. Using incomplete candles will lead to repainting and consequently to strategies with "ghost" buys, which are impossible to both backtest, and verify after they happened.

docs/freqai-feature-engineering.md

@@ -178,7 +178,7 @@ You can ask for each of the defined features to be included also for informative
 
 `include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells FreqAI to include the past 2 candles for each of the features in the feature set.
 
-In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles`
+In total, the number of features the user of the presented example strategy has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles`
  $= 3 * 3 * 3 * 2 * 2 = 108$.
  
  !!! note "Learn more about creative feature engineering"

docs/freqai-parameter-table.md

@@ -100,12 +100,12 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 
 #### trainer_kwargs
 
-|  Parameter | Description |
-|------------|-------------|
-|  |  **Model training parameters within the `freqai.model_training_parameters.model_kwargs` sub dictionary**
-| `max_iters` | The number of training iterations to run. iteration here refers to the number of times we call self.optimizer.step(). used to calculate n_epochs. <br> **Datatype:** int. <br> Default: `100`.
-| `batch_size` | The size of the batches to use during training.. <br> **Datatype:** int. <br> Default: `64`.
-| `max_n_eval_batches` | The maximum number batches to use for evaluation.. <br> **Datatype:** int, optional. <br> Default: `None`.
+| Parameter    | Description |
+|--------------|-------------|
+|              |  **Model training parameters within the `freqai.model_training_parameters.model_kwargs` sub dictionary**
+| `n_epochs`   | The `n_epochs` parameter is a crucial setting in the PyTorch training loop that determines the number of times the entire training dataset will be used to update the model's parameters. An epoch represents one full pass through the entire training dataset. Overrides `n_steps`. Either `n_epochs` or `n_steps` must be set. <br><br> **Datatype:** int. optional. <br> Default: `10`.
+| `n_steps`    | An alternative way of setting `n_epochs` -  the number of training iterations to run. Iteration here refer to the number of times we call `optimizer.step()`. Ignored if `n_epochs` is set. A simplified version of the function: <br><br> n_epochs = n_steps / (n_obs / batch_size) <br><br> The motivation here is that `n_steps` is easier to optimize and keep stable across different n_obs - the number of data points.  <br> <br> **Datatype:** int. optional. <br> Default: `None`.
+| `batch_size` | The size of the batches to use during training. <br><br> **Datatype:** int. <br> Default: `64`.
 
 
 ### Additional parameters

docs/freqai-reinforcement-learning.md

@@ -20,7 +20,7 @@ With the current framework, we aim to expose the training environment via the co
 
 We envision the majority of users focusing their effort on creative design of the `calculate_reward()` function [details here](#creating-a-custom-reward-function), while leaving the rest of the environment untouched. Other users may not touch the environment at all, and they will only play with the configuration settings and the powerful feature engineering that already exists in FreqAI. Meanwhile, we enable advanced users to create their own model classes entirely.
 
-The framework is built on stable_baselines3 (torch) and OpenAI gym for the base environment class. But generally speaking, the model class is well isolated. Thus, the addition of competing libraries can be easily integrated into the existing framework. For the environment, it is inheriting from `gym.env` which means that it is necessary to write an entirely new environment in order to switch to a different library.
+The framework is built on stable_baselines3 (torch) and OpenAI gym for the base environment class. But generally speaking, the model class is well isolated. Thus, the addition of competing libraries can be easily integrated into the existing framework. For the environment, it is inheriting from `gym.Env` which means that it is necessary to write an entirely new environment in order to switch to a different library.
 
 ### Important considerations
 
@@ -173,7 +173,7 @@ class MyCoolRLModel(ReinforcementLearner):
     """
     class MyRLEnv(Base5ActionRLEnv):
         """
-        User made custom environment. This class inherits from BaseEnvironment and gym.env.
+        User made custom environment. This class inherits from BaseEnvironment and gym.Env.
         Users can override any functions from those parent classes. Here is an example
         of a user customized `calculate_reward()` function.
 
@@ -237,11 +237,10 @@ class MyCoolRLModel(ReinforcementLearner):
 Reinforcement Learning models benefit from tracking training metrics. FreqAI has integrated Tensorboard to allow users to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command:
 
 ```bash
-cd freqtrade
 tensorboard --logdir user_data/models/unique-id
 ```
 
-where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard).
+where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in the browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard).
 
 ![tensorboard](assets/tensorboard.jpg)
 
@@ -254,7 +253,7 @@ FreqAI also provides a built in episodic summary logger called `self.tensorboard
 ```python
     class MyRLEnv(Base5ActionRLEnv):
         """
-        User made custom environment. This class inherits from BaseEnvironment and gym.env.
+        User made custom environment. This class inherits from BaseEnvironment and gym.Env.
         Users can override any functions from those parent classes. Here is an example
         of a user customized `calculate_reward()` function.
         """

docs/hyperopt.md

@@ -31,7 +31,7 @@ The docker-image includes hyperopt dependencies, no further action needed.
 ### Easy installation script (setup.sh) / Manual installation
 
 ```bash
-source .env/bin/activate
+source .venv/bin/activate
 pip install -r requirements-hyperopt.txt
 ```
 
@@ -433,9 +433,14 @@ While this strategy is most likely too simple to provide consistent profit, it s
     `range` property may also be used with `DecimalParameter` and `CategoricalParameter`. `RealParameter` does not provide this property due to infinite search space.
 
 ??? Hint "Performance tip"
-    During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. This will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
+    During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage
 
-    In either case, you should try to use space ranges as small as possible this will improve CPU/RAM usage in both scenarios.
+    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality.
+    * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value.
+
+    These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
+
+    Whether you are using `.range` functionality or the alternatives above, you should try to use space ranges as small as possible since this will improve CPU/RAM usage.
 
 
 ## Optimizing protections

docs/includes/pairlists.md

@@ -25,6 +25,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`ProducerPairList`](#producerpairlist)
 * [`RemotePairList`](#remotepairlist)
 * [`AgeFilter`](#agefilter)
+* [`FullTradesFilter`](#fulltradesfilter)
 * [`OffsetFilter`](#offsetfilter)
 * [`PerformanceFilter`](#performancefilter)
 * [`PrecisionFilter`](#precisionfilter)
@@ -236,6 +237,17 @@ be caught out buying before the pair has finished dropping in price.
 
 This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`.
 
+#### FullTradesFilter
+
+Shrink whitelist to consist only in-trade pairs when the trade slots are full (when `max_open_trades` isn't being set to `-1` in the config).
+
+When the trade slots are full, there is no need to calculate indicators of the rest of the pairs (except informative pairs) since no new trade can be opened. By shrinking the whitelist to just the in-trade pairs, you can improve calculation speeds and reduce CPU usage. When a trade slot is free (either a trade is closed or `max_open_trades` value in config is increased), then the whitelist will return to normal state.
+
+When multiple pairlist filters are being used, it's recommended to put this filter at second position directly below the primary pairlist, so when the trade slots are full, the bot doesn't have to download data for the rest of the filters.
+
+!!! Warning "Backtesting"
+    `FullTradesFilter` does not support backtesting mode.
+
 #### OffsetFilter
 
 Offsets an incoming pairlist by a given `offset` value.
@@ -376,7 +388,7 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from
         "lookback_days": 10,
         "min_rate_of_change": 0.01,
         "max_rate_of_change": 0.99,
-        "refresh_period": 1440
+        "refresh_period": 86400
     }
 ]
 ```
@@ -431,7 +443,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "method": "RangeStabilityFilter",
         "lookback_days": 10,
         "min_rate_of_change": 0.01,
-        "refresh_period": 1440
+        "refresh_period": 86400
     },
     {
         "method": "VolatilityFilter",

docs/index.md

@@ -83,7 +83,7 @@ To run this bot we recommend you a linux cloud instance with a minimum of:
 
 Alternatively
 
-- Python 3.8+
+- Python 3.9+
 - pip (pip3)
 - git
 - TA-Lib

docs/installation.md

@@ -24,7 +24,7 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito
     The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable).
 
 !!! Note
-    Python3.8 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository.  
+    Python3.9 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository.  
     Also, python headers (`python<yourversion>-dev` / `python<yourversion>-devel`) must be available for the installation to complete successfully.
 
 !!! Warning "Up-to-date clock"
@@ -42,7 +42,7 @@ These requirements apply to both [Script Installation](#script-installation) and
 
 ### Install guide
 
-* [Python >= 3.8.x](http://docs.python-guide.org/en/latest/starting/installation/)
+* [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/)
 * [pip](https://pip.pypa.io/en/stable/installing/)
 * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
 * [virtualenv](https://virtualenv.pypa.io/en/stable/installation.html) (Recommended)
@@ -54,7 +54,7 @@ We've included/collected install instructions for Ubuntu, MacOS, and Windows. Th
 OS Specific steps are listed first, the [Common](#common) section below is necessary for all systems.
 
 !!! Note
-    Python3.8 or higher and the corresponding pip are assumed to be available.
+    Python3.9 or higher and the corresponding pip are assumed to be available.
 
 === "Debian/Ubuntu"
     #### Install necessary dependencies
@@ -143,11 +143,11 @@ If you are on Debian, Ubuntu or MacOS, freqtrade provides the script to install
 
 ### Activate your virtual environment
 
-Each time you open a new terminal, you must run `source .env/bin/activate` to activate your virtual environment.
+Each time you open a new terminal, you must run `source .venv/bin/activate` to activate your virtual environment.
 
 ```bash
-# then activate your .env
-source ./.env/bin/activate
+# activate virtual environment
+source ./.venv/bin/activate
 ```
 
 ### Congratulations
@@ -169,10 +169,10 @@ You can as well update, configure and reset the codebase of your bot with `./scr
 ** --install **
 
 With this option, the script will install the bot and most dependencies:
-You will need to have git and python3.8+ installed beforehand for this to work.
+You will need to have git and python3.9+ installed beforehand for this to work.
 
 * Mandatory software as: `ta-lib`
-* Setup your virtualenv under `.env/`
+* Setup your virtualenv under `.venv/`
 
 This option is a combination of installation tasks and `--reset`
 
@@ -225,11 +225,11 @@ rm -rf ./ta-lib*
 You will run freqtrade in separated `virtual environment`
 
 ```bash
-# create virtualenv in directory /freqtrade/.env
-python3 -m venv .env
+# create virtualenv in directory /freqtrade/.venv
+python3 -m venv .venv
 
 # run virtualenv
-source .env/bin/activate
+source .venv/bin/activate
 ```
 
 #### Install python dependencies
@@ -286,7 +286,7 @@ cd freqtrade
 #### Freqtrade install: Conda Environment
 
 ```bash
-conda create --name freqtrade python=3.10
+conda create --name freqtrade python=3.11
 ```
 
 !!! Note "Creating Conda Environment"
@@ -383,7 +383,7 @@ You've made it this far, so you have successfully installed freqtrade.
 freqtrade create-userdir --userdir user_data
 
 # Step 2 - Create a new configuration file
-freqtrade new-config --config config.json
+freqtrade new-config --config user_data/config.json
 ```
 
 You are ready to run, read [Bot Configuration](configuration.md), remember to start with `dry_run: True` and verify that everything is working.
@@ -393,7 +393,7 @@ To learn how to setup your configuration, please refer to the [Bot Configuration
 ### Start the Bot
 
 ```bash
-freqtrade trade --config config.json --strategy SampleStrategy
+freqtrade trade --config user_data/config.json --strategy SampleStrategy
 ```
 
 !!! Warning
@@ -411,8 +411,8 @@ If you used (1)`Script` or (2)`Manual` installation, you need to run the bot in
 # if:
 bash: freqtrade: command not found
 
-# then activate your .env
-source ./.env/bin/activate
+# then activate your virtual environment
+source ./.venv/bin/activate
 ```
 
 ### MacOS installation error

docs/leverage.md

@@ -64,7 +64,7 @@ You will also have to pick a "margin mode" (explanation below) - with freqtrade
 
 ##### Pair namings
 
-Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/en/latest/manual.html?#perpetual-swap-perpetual-future).
+Freqtrade follows the [ccxt naming conventions for futures](https://docs.ccxt.com/#/README?id=perpetual-swap-perpetual-future).
 A futures pair will therefore have the naming of `base/quote:settle` (e.g. `ETH/USDT:USDT`).
 
 ### Margin mode

docs/lookahead-analysis.md

@@ -21,7 +21,10 @@ It also supports the lookahead-analysis of freqai strategies.
 
 - `--cache` is forced to "none".
 - `--max-open-trades` is forced to be at least equal to the number of pairs.
-- `--dry-run-wallet` is forced to be basically infinite.
+- `--dry-run-wallet` is forced to be basically infinite (1 billion).
+- `--stake-amount` is forced to be a static 10000 (10k).
+
+Those are set to avoid users accidentally generating false positives.
 
 ## Lookahead-analysis command reference
 

docs/recursive-analysis.md

@@ -0,0 +1,89 @@
+# Recursive analysis
+
+This page explains how to validate your strategy for inaccuracies due to recursive issues with certain indicators.
+
+A recursive formula defines any term of a sequence relative to its preceding term(s). An example of a recursive formula is a<sub>n</sub> = a<sub>n-1</sub> + b.
+
+Why does this matter for Freqtrade? In backtesting, the bot will get full data of the pairs according to the timerange specified. But in a dry/live run, the bot will be limited by the amount of data each exchanges gives.
+
+For example, to calculate a very basic indicator called `steps`, the first row's value is always 0, while the following rows' values are equal to the value of the previous row plus 1. If I were to calculate it using the latest 1000 candles, then the `steps` value of the first row is 0, and the `steps` value at the last closed candle is 999.
+
+What happens if the calculation is using only the latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result.
+
+The `recursive-analysis` command requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in,
+head over to the [Data Downloading](data-download.md) section of the documentation.
+
+This command is built upon preparing different lengths of data and calculates indicators based on them.
+This does not backtest the strategy itself, but rather only calculates the indicators. After calculating the indicators of different startup candle values (`startup_candle_count`) are done, the values of last rows across all specified `startup_candle_count` are compared to see how much variance they show compared to the base calculation.
+
+Command settings:
+
+- Use the `-p` option to set your desired pair to analyze. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist.
+- It is recommended to set a long timerange (at least 5000 candles) so that the initial indicators' calculation that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days.
+- `--cache` is forced to "none" to avoid loading previous indicators calculation automatically.
+
+In addition to the recursive formula check, this command also carries out a simple lookahead bias check on the indicator values only. For a full lookahead check, use [Lookahead-analysis](lookahead-analysis.md).
+
+## Recursive-analysis command reference
+
+```
+usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH]
+                                    [-d PATH] [--userdir PATH] [-s NAME]
+                                    [--strategy-path PATH]
+                                    [--recursive-strategy-search]
+                                    [--freqaimodel NAME]
+                                    [--freqaimodel-path PATH] [-i TIMEFRAME]
+                                    [--timerange TIMERANGE]
+                                    [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
+                                    [-p PAIR]
+                                    [--freqai-backtest-live-models]
+                                    [--startup-candle STARTUP_CANDLES [STARTUP_CANDLES ...]]
+
+optional arguments:
+-p PAIR, --pairs PAIR
+                        Limit command to this pair.
+--startup-candle STARTUP_CANDLE [STARTUP_CANDLE ...]
+                        Provide a space-separated list of startup_candle_count to
+                        be checked. Default : `199 399 499 999 1999`.
+```
+
+### Why are odd-numbered default startup candles used?
+
+The default value for startup candles are odd numbers. When the bot fetches candle data from the exchange's API, the last candle is the one being checked by the bot and the rest of the data are the "startup candles".
+
+For example, Binance allows 1000 candles per API call. When the bot receives 1000 candles, the last candle is the "current candle", and the preceding 999 candles are the "startup candles". By setting the startup candle count as 1000 instead of 999, the bot will try to fetch 1001 candles instead. The exchange API will then send candle data in a paginated form, i.e. in case of the Binance API, this will be two groups- one of length 1000 and another of length 1. This results in the bot thinking the strategy needs 1001 candles of data, and so it will download 2000 candles worth of data instead, which means there will be 1 "current candle" and 1999 "startup candles".
+
+Furthermore, exchanges limit the number of consecutive bulk API calls, e.g. Binance allows 5 calls. In this case, only 5000 candles can be downloaded from Binance API without hitting the API rate limit, which means the max `startup_candle_count` you can have is 4999.
+
+Please note that this candle limit may be changed in the future by the exchanges without any prior notice.
+
+### How does the command work?
+
+- Firstly an initial indicator calculation is carried out using the supplied timerange to generate a benchmark for indicator values.
+- After setting the benchmark it will then carry out additional runs for each of the different startup candle count values.
+- The command will then compare the indicator values at the last candle rows and report the differences in a table.
+
+## Understanding the recursive-analysis output
+
+This is an example of an output results table where at least one indicator has a recursive formula issue:
+
+```
+| indicators   | 20      | 40      | 80     | 100    | 150     | 300     | 999    |
+|--------------+---------+---------+--------+--------+---------+---------+--------|
+| rsi_30       | nan%    | -6.025% | 0.612% | 0.828% | -0.140% | 0.000%  | 0.000% |
+| rsi_14       | 24.141% | -0.876% | 0.070% | 0.007% | -0.000% | -0.000% | -      |
+```
+
+The column headers indicate the different `startup_candle_count` used in the analysis. The values in the table indicate the variance of the calculated indicators compared to the benchmark value.
+
+`nan%` means the value of that indicator cannot be calculated due to lack of data. In this example, you cannot calculate RSI with length 30 with just 21 candles (1 current candle + 20 startup candles).
+
+Users should assess the table per indicator to decide if the specified `startup_candle_count` results in a sufficiently small variance so that the indicator does not have any effect on entries and/or exits.
+
+As such, aiming for absolute zero variance (shown by `-` value) might not be the best option, because some indicators might require you to use such a long `startup_candle_count` to have zero variance.
+
+## Caveats
+
+- `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count calculations and the original benchmark calculation. Whether it has any actual impact on your entries and exits is not included.
+- The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and therefore `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions.
+- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated.

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
-markdown==3.3.7
-mkdocs==1.4.3
-mkdocs-material==9.1.19
+markdown==3.4.4
+mkdocs==1.5.3
+mkdocs-material==9.4.1
 mdx_truly_sane_lists==1.3
-pymdown-extensions==10.1
+pymdown-extensions==10.3
 jinja2==3.1.2

docs/rest-api.md

@@ -151,6 +151,8 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 | `performance` | Show performance of each finished trade grouped by pair.
 | `balance` | Show account balance per currency.
 | `daily <n>` | Shows profit or loss per day, over the last n days (n defaults to 7).
+| `weekly <n>` | Shows profit or loss per week, over the last n days (n defaults to 4).
+| `monthly <n>` | Shows profit or loss per month, over the last n days (n defaults to 3).
 | `stats` | Display a summary of profit / loss reasons as well as average holding times.
 | `whitelist` | Show the current whitelist.
 | `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist.

docs/sandbox-testing.md

@@ -1,121 +0,0 @@
-# Sandbox API testing
-
-Some exchanges provide sandboxes or testbeds for risk-free testing, while running the bot against a real exchange.
-With some configuration, freqtrade (in combination with ccxt) provides access to these.
-
-This document is an overview to configure Freqtrade to be used with sandboxes.
-This can be useful to developers and trader alike.
-
-!!! Warning
-    Sandboxes usually have very low volume, and either a very wide spread, or no orders available at all.
-    Therefore, sandboxes will usually not do a good job of showing you how a strategy would work in real trading.
-
-## Exchanges known to have a sandbox / testnet
-
-* [binance](https://testnet.binance.vision/)
-* [coinbasepro](https://public.sandbox.pro.coinbase.com)
-* [gemini](https://exchange.sandbox.gemini.com/)
-* [huobipro](https://www.testnet.huobi.pro/)
-* [kucoin](https://sandbox.kucoin.com/)
-* [phemex](https://testnet.phemex.com/) 
-
-!!! Note
-    We did not test correct functioning of all of the above testnets. Please report your experiences with each sandbox.
-
----
-
-## Configure a Sandbox account
-
-When testing your API connectivity, make sure to use the appropriate sandbox / testnet URL.
-
-In general, you should follow these steps to enable an exchange's sandbox:
-
-* Figure out if an exchange has a sandbox (most likely by using google or the exchange's support documents)
-* Create a sandbox account (often the sandbox-account requires separate registration)
-* [Add some test assets to account](#add-test-funds)
-* Create API keys
-
-### Add test funds
-
-Usually, sandbox exchanges allow depositing funds directly via web-interface.
-You should make sure to have a realistic amount of funds available to your test-account, so results are representable of your real account funds.
-
-!!! Warning
-    Test exchanges will **NEVER** require your real credit card or banking details!
-
-## Configure freqtrade to use a exchange's sandbox
-
-### Sandbox URLs
-
-Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade.
-These include `['test']` and `['api']`.
-
-* `[Test]` if available will point to an Exchanges sandbox.
-* `[Api]` normally used, and resolves to live API target on the exchange.
-
-To make use of sandbox / test add "sandbox": true, to your config.json
-
-```json
-  "exchange": {
-        "name": "coinbasepro",
-        "sandbox": true,
-        "key": "5wowfxemogxeowo;heiohgmd",
-        "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==",
-        "password": "1bkjfkhfhfu6sr",
-        "outdated_offset": 5
-        "pair_whitelist": [
-            "BTC/USD"
-        ]
-  },
-  "datadir": "user_data/data/coinbasepro_sandbox"
-```
-
-Also the following information:
-
-* api-key (created for the sandbox webpage)
-* api-secret (noted earlier)
-* password (the passphrase - noted earlier)
-
-!!! Tip "Different data directory"
-    We also recommend to set `datadir` to something identifying downloaded data as sandbox data, to avoid having sandbox data mixed with data from the real exchange.
-    This can be done by adding the `"datadir"` key to the configuration.
-    Now, whenever you use this configuration, your data directory will be set to this directory.
-
----
-
-## You should now be ready to test your sandbox
-
-Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. Also make sure to select a pair which shows at least some decent value (which very often is BTC/<somestablecoin>).
-
-## Common problems with sandbox exchanges
-
-Sandbox exchange instances often have very low volume, which can cause some problems which usually are not seen on a real exchange instance.
-
-### Old Candles problem
-
-Since Sandboxes often have low volume, candles can be quite old and show no volume.
-To disable the error "Outdated history for pair ...", best increase the parameter `"outdated_offset"` to a number that seems realistic for the sandbox you're using.
-
-### Unfilled orders
-
-Sandboxes often have very low volumes - which means that many trades can go unfilled, or can go unfilled for a very long time.
-
-To mitigate this, you can try to match the first order on the opposite orderbook side using the following configuration:
-
-``` jsonc
-  "order_types": {
-    "entry": "limit",
-    "exit": "limit"
-    // ...
-  },
-  "entry_pricing": {
-    "price_side": "other",
-    // ...
-  },
-  "exit_pricing":{
-    "price_side": "other",
-    // ...
-  },
-  ```
-
-  The configuration is similar to the suggested configuration for market orders - however by using limit-orders you can avoid moving the price too much, and you can set the worst price you might get.

docs/strategy-callbacks.md

@@ -164,6 +164,31 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo
 During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades).
 
 The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price.
+Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss.
+
+Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)).
+
+!!! Note "Use of dates"
+    All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support.
+
+!!! Tip "Trailing stoploss"
+    It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior.
+
+### Adjust stoploss after position adjustments
+
+Depending on your strategy, you may encounter the need to adjust the stoploss in both directions after a [position adjustment](#adjust-trade-position).
+For this, freqtrade will make an additional call with `after_fill=True` after an order fills, which will allow the strategy to move the stoploss in any direction (also widening the gap between stoploss and current price, which is otherwise forbidden).
+
+!!! Note "backwards compatibility"
+    This call will only be made if the `after_fill` parameter is part of the function definition of your `custom_stoploss` function.
+    As such, this will not impact (and with that, surprise) existing, running strategies.
+
+### Custom stoploss examples
+
+The next section will show some examples on what's possible with the custom stoploss function.
+Of course, many more things are possible, and all examples can be combined at will.
+
+#### Trailing stop via custom stoploss
 
 To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum reached price) you would use the following very simple method:
 
@@ -179,7 +204,8 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool, 
+                        **kwargs) -> Optional[float]:
         """
         Custom stoploss logic, returning the new distance relative to current_rate (as ratio).
         e.g. returning -0.05 would create a stoploss 5% below current_rate.
@@ -187,7 +213,7 @@ class AwesomeStrategy(IStrategy):
 
         For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
 
-        When not implemented by a strategy, returns the initial stoploss value
+        When not implemented by a strategy, returns the initial stoploss value.
         Only called when use_custom_stoploss is set to True.
 
         :param pair: Pair that's currently analyzed
@@ -195,25 +221,13 @@ class AwesomeStrategy(IStrategy):
         :param current_time: datetime object, containing the current datetime
         :param current_rate: Rate, calculated based on pricing settings in exit_pricing.
         :param current_profit: Current profit (as ratio), calculated based on current_rate.
+        :param after_fill: True if the stoploss is called after the order was filled.
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
-        :return float: New stoploss value, relative to the current rate
+        :return float: New stoploss value, relative to the current_rate
         """
         return -0.04
 ```
 
-Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)).
-
-!!! Note "Use of dates"
-    All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support.
-
-!!! Tip "Trailing stoploss"
-    It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior.
-
-### Custom stoploss examples
-
-The next section will show some examples on what's possible with the custom stoploss function.
-Of course, many more things are possible, and all examples can be combined at will.
-
 #### Time based trailing stop
 
 Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss.
@@ -229,14 +243,45 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool, 
+                        **kwargs) -> Optional[float]:
 
         # Make sure you have the longest interval first - these conditions are evaluated from top to bottom.
         if current_time - timedelta(minutes=120) > trade.open_date_utc:
             return -0.05
         elif current_time - timedelta(minutes=60) > trade.open_date_utc:
             return -0.10
-        return 1
+        return None
+```
+
+#### Time based trailing stop with after-fill adjustments
+
+Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss.
+If an additional order fills, set stoploss to -10% below the new `open_rate` ([Averaged across all entries](#position-adjust-calculations)).
+
+``` python
+from datetime import datetime, timedelta
+from freqtrade.persistence import Trade
+
+class AwesomeStrategy(IStrategy):
+
+    # ... populate_* methods
+
+    use_custom_stoploss = True
+
+    def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
+                        current_rate: float, current_profit: float, after_fill: bool, 
+                        **kwargs) -> Optional[float]:
+
+        if after_fill: 
+            # After an additional order, start with a stoploss of 10% below the new open rate
+            return stoploss_from_open(0.10, current_profit, is_short=trade.is_short, leverage=trade.leverage)
+        # Make sure you have the longest interval first - these conditions are evaluated from top to bottom.
+        if current_time - timedelta(minutes=120) > trade.open_date_utc:
+            return -0.05
+        elif current_time - timedelta(minutes=60) > trade.open_date_utc:
+            return -0.10
+        return None
 ```
 
 #### Different stoploss per pair
@@ -255,7 +300,8 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool,
+                        **kwargs) -> Optional[float]:
 
         if pair in ('ETH/BTC', 'XRP/BTC'):
             return -0.10
@@ -281,7 +327,8 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool,
+                        **kwargs) -> Optional[float]:
 
         if current_profit < 0.04:
             return -1 # return a value bigger than the initial stoploss to keep using the initial stoploss
@@ -314,7 +361,8 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool,
+                        **kwargs) -> Optional[float]:
 
         # evaluate highest to lowest, so that highest possible stop is used
         if current_profit > 0.40:
@@ -325,7 +373,7 @@ class AwesomeStrategy(IStrategy):
             return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage)
 
         # return maximum stoploss value, keeping current stoploss price unchanged
-        return 1
+        return None
 ```
 
 #### Custom stoploss using an indicator from dataframe example
@@ -342,7 +390,8 @@ class AwesomeStrategy(IStrategy):
     use_custom_stoploss = True
 
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool,
+                        **kwargs) -> Optional[float]:
 
         dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
         last_candle = dataframe.iloc[-1].squeeze()
@@ -355,7 +404,7 @@ class AwesomeStrategy(IStrategy):
             return stoploss_from_absolute(stoploss_price, current_rate, is_short=trade.is_short)
 
         # return maximum stoploss value, keeping current stoploss price unchanged
-        return 1
+        return None
 ```
 
 See [Dataframe access](strategy-advanced.md#dataframe-access) for more information about dataframe use in strategy callbacks.
@@ -364,15 +413,89 @@ See [Dataframe access](strategy-advanced.md#dataframe-access) for more informati
 
 #### Stoploss relative to open price
 
-Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss relative to the *open* price, we need to use `current_profit` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price.
+Stoploss values returned from `custom_stoploss()` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the _entry_ price instead.
+`stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point.
 
-The helper function [`stoploss_from_open()`](strategy-customization.md#stoploss_from_open) can be used to convert from an open price relative stop, to a current price relative stop which can be returned from `custom_stoploss()`.
+??? Example "Returning a stoploss relative to the open price from the custom stoploss function"
+
+    Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`).  
+
+    If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`.  11.57% below $121 is $107, which is the same as 7% above $100.
+
+    This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%).
+
+
+    ``` python
+
+    from datetime import datetime
+    from freqtrade.persistence import Trade
+    from freqtrade.strategy import IStrategy, stoploss_from_open
+
+    class AwesomeStrategy(IStrategy):
+
+        # ... populate_* methods
+
+        use_custom_stoploss = True
+
+        def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
+                            current_rate: float, current_profit: float, after_fill: bool,
+                            **kwargs) -> Optional[float]:
+
+            # once the profit has risen above 10%, keep the stoploss at 7% above the open price
+            if current_profit > 0.10:
+                return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage)
+
+            return 1
+
+    ```
+
+    Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation.
+
+!!! Note
+    Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings.
+    This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade
+    is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `exit_reason` in
+    `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when
+    `current_profit < open_relative_stop`.
 
 #### Stoploss percentage from absolute price
 
 Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss at specified absolute price level, we need to use `stop_rate` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price.
 
-The helper function [`stoploss_from_absolute()`](strategy-customization.md#stoploss_from_absolute) can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`.
+The helper function `stoploss_from_absolute()` can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`.
+
+??? Example "Returning a stoploss using absolute price from the custom stoploss function"
+
+    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage)`.
+    For futures, we need to adjust the direction (up or down), as well as adjust for leverage, since the [`custom_stoploss`](strategy-callbacks.md#custom-stoploss) callback  returns the ["risk for this trade"](stoploss.md#stoploss-and-leverage) - not the relative price movement.
+
+    ``` python
+
+    from datetime import datetime
+    from freqtrade.persistence import Trade
+    from freqtrade.strategy import IStrategy, stoploss_from_absolute, timeframe_to_prev_date
+
+    class AwesomeStrategy(IStrategy):
+
+        use_custom_stoploss = True
+
+        def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+            dataframe['atr'] = ta.ATR(dataframe, timeperiod=14)
+            return dataframe
+
+        def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
+                            current_rate: float, current_profit: float, after_fill: bool,
+                            **kwargs) -> Optional[float]:
+            dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
+            trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
+            candle = dataframe.iloc[-1].squeeze()
+            sign = 1 if trade.is_short else -1
+            return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), 
+                                          current_rate, is_short=trade.is_short,
+                                          leverage=trade.leverage)
+
+    ```
+
 
 ---
 
@@ -387,6 +510,9 @@ Each of these methods are called right before placing an order on the exchange.
 !!! Note
     If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration.
 
+!!! Note
+    Using custom_entry_price, the Trade object will be available as soon as the first entry order associated with the trade is created, for the first entry, `trade` parameter value will be `None`.
+
 ### Custom order entry and exit price example
 
 ``` python
@@ -397,7 +523,7 @@ class AwesomeStrategy(IStrategy):
 
     # ... populate_* methods
 
-    def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float,
+    def custom_entry_price(self, pair: str, trade: Optional['Trade'], current_time: datetime, proposed_rate: float,
                            entry_tag: Optional[str], side: str, **kwargs) -> float:
 
         dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
@@ -700,7 +826,7 @@ class DigDeeperStrategy(IStrategy):
         """
         Custom trade adjustment logic, returning the stake amount that a trade should be
         increased or decreased.
-        This means extra buy or sell orders with additional fees.
+        This means extra entry or exit orders with additional fees.
         Only called when `position_adjustment_enable` is set to True.
 
         For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
@@ -709,8 +835,9 @@ class DigDeeperStrategy(IStrategy):
 
         :param trade: trade object.
         :param current_time: datetime object, containing the current datetime
-        :param current_rate: Current buy rate.
-        :param current_profit: Current profit (as ratio), calculated based on current_rate.
+        :param current_rate: Current entry rate (same as current_entry_profit)
+        :param current_profit: Current profit (as ratio), calculated based on current_rate 
+                               (same as current_entry_profit).
         :param min_stake: Minimal stake size allowed by exchange (for both entries and exits)
         :param max_stake: Maximum stake allowed (either through balance, or by exchange limits).
         :param current_entry_rate: Current rate using entry pricing.
@@ -793,6 +920,8 @@ Returning any other price will cancel the existing order, and replace it with a
 The trade open-date (`trade.open_date_utc`) will remain at the time of the very first order placed.
 Please make sure to be aware of this - and eventually adjust your logic in other callbacks to account for this, and use the date of the first filled order instead.
 
+If the cancellation of the original order fails, then the order will not be replaced - though the order will most likely have been canceled on exchange. Having this happen on initial entries will result in the deletion of the order, while on position adjustment orders, it'll result in the trade size remaining as is.
+
 !!! Warning "Regular timeout"
     Entry `unfilledtimeout` mechanism (as well as `check_entry_timeout()`) takes precedence over this.
     Entry Orders that are cancelled via the above methods will not have this callback called. Be sure to update timeout values to match your expectations.

docs/strategy-customization.md

@@ -168,10 +168,12 @@ Most indicators have an instable startup period, in which they are either not av
 To account for this, the strategy can be assigned the `startup_candle_count` attribute.
 This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators.
 
-In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles.
+You can use [recursive-analysis](recursive-analysis.md) to check and find the correct `startup_candle_count` to be used.
+
+In this example strategy, this should be set to 400 (`startup_candle_count = 400`), since the minimum needed history for ema100 calculation to make sure the value is correct is 400 candles.
 
 ``` python
-    dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100)
+    dataframe['ema100'] = ta.EMA(dataframe, timeperiod=400)
 ```
 
 By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt.
@@ -193,11 +195,11 @@ Let's try to backtest 1 month (January 2019) of 5m candles using an example stra
 freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m
 ```
 
-Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00.
+Assuming `startup_candle_count` is set to 400, backtesting knows it needs 400 candles to generate valid buy signals. It will load data from `20190101 - (400 * 5m)` - which is ~2018-12-30 11:40:00.
 If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting.
 
 !!! Note
-    If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00.
+    If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-02 09:20:00.
 
 ### Entry signal rules
 
@@ -264,7 +266,7 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram
 ### Exit signal rules
 
 Edit the method `populate_exit_trend()` into your strategy file to update your exit strategy.
-The exit-signal is only used for exits if `use_exit_signal` is set to true in the configuration.
+The exit-signal can be suppressed by setting `use_exit_signal` to false in the configuration or strategy.
 `use_exit_signal` will not influence [signal collision rules](#colliding-signals) - which will still apply and can prevent entries.
 
 It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
@@ -586,6 +588,67 @@ for more information.
     will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators
     created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique!
 
+### *merge_informative_pair()*
+
+This method helps you merge an informative pair to a regular dataframe without lookahead bias.
+It's there to help you merge the dataframe in a safe and consistent way.
+
+Options:
+
+- Rename the columns for you to create unique columns
+- Merge the dataframe without lookahead bias
+- Forward-fill (optional)
+
+For a full sample, please refer to the [complete data provider example](#complete-data-provider-sample) below.
+
+All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion:
+
+!!! Example "Column renaming"
+    Assuming `inf_tf = '1d'` the resulting columns will be:
+
+    ``` python
+    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
+    'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d'   # from the informative dataframe
+    ```
+
+??? Example "Column renaming - 1h"
+    Assuming `inf_tf = '1h'` the resulting columns will be:
+
+    ``` python
+    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
+    'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h'   # from the informative dataframe
+    ```
+
+??? Example "Custom implementation"
+    A custom implementation for this is possible, and can be done as follows:
+
+    ``` python
+
+    # Shift date by 1 candle
+    # This is necessary since the data is always the "open date"
+    # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00
+    minutes = timeframe_to_minutes(inf_tf)
+    # Only do this if the timeframes are different:
+    informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm')
+
+    # Rename columns to be unique
+    informative.columns = [f"{col}_{inf_tf}" for col in informative.columns]
+    # Assuming inf_tf = '1d' - then the columns will now be:
+    # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
+
+    # Combine the 2 dataframes
+    # all indicators on the informative sample MUST be calculated before this point
+    dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left')
+    # FFill to have the 1d value available in every row throughout the day.
+    # Without this, comparisons would only work once per day.
+    dataframe = dataframe.ffill()
+
+    ```
+
+!!! Warning "Informative timeframe < timeframe"
+    Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide.
+    To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need).
+
 ## Additional data (DataProvider)
 
 The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy.
@@ -810,146 +873,6 @@ class SampleStrategy(IStrategy):
 
 ***
 
-## Helper functions
-
-### *merge_informative_pair()*
-
-This method helps you merge an informative pair to a regular dataframe without lookahead bias.
-It's there to help you merge the dataframe in a safe and consistent way.
-
-Options:
-
-- Rename the columns for you to create unique columns
-- Merge the dataframe without lookahead bias
-- Forward-fill (optional)
-
-For a full sample, please refer to the [complete data provider example](#complete-data-provider-sample) below.
-
-All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion:
-
-!!! Example "Column renaming"
-    Assuming `inf_tf = '1d'` the resulting columns will be:
-
-    ``` python
-    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
-    'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d'   # from the informative dataframe
-    ```
-
-??? Example "Column renaming - 1h"
-    Assuming `inf_tf = '1h'` the resulting columns will be:
-
-    ``` python
-    'date', 'open', 'high', 'low', 'close', 'rsi'                     # from the original dataframe
-    'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h'   # from the informative dataframe
-    ```
-
-??? Example "Custom implementation"
-    A custom implementation for this is possible, and can be done as follows:
-
-    ``` python
-
-    # Shift date by 1 candle
-    # This is necessary since the data is always the "open date"
-    # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00
-    minutes = timeframe_to_minutes(inf_tf)
-    # Only do this if the timeframes are different:
-    informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm')
-
-    # Rename columns to be unique
-    informative.columns = [f"{col}_{inf_tf}" for col in informative.columns]
-    # Assuming inf_tf = '1d' - then the columns will now be:
-    # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
-
-    # Combine the 2 dataframes
-    # all indicators on the informative sample MUST be calculated before this point
-    dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left')
-    # FFill to have the 1d value available in every row throughout the day.
-    # Without this, comparisons would only work once per day.
-    dataframe = dataframe.ffill()
-
-    ```
-
-!!! Warning "Informative timeframe < timeframe"
-    Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide.
-    To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need).
-
-***
-
-### *stoploss_from_open()*
-
-Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the entry point instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point.
-
-??? Example "Returning a stoploss relative to the open price from the custom stoploss function"
-
-    Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`).  
-
-    If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`.  11.57% below $121 is $107, which is the same as 7% above $100.
-
-    This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%).
-
-
-    ``` python
-
-    from datetime import datetime
-    from freqtrade.persistence import Trade
-    from freqtrade.strategy import IStrategy, stoploss_from_open
-
-    class AwesomeStrategy(IStrategy):
-
-        # ... populate_* methods
-
-        use_custom_stoploss = True
-
-        def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                            current_rate: float, current_profit: float, **kwargs) -> float:
-
-            # once the profit has risen above 10%, keep the stoploss at 7% above the open price
-            if current_profit > 0.10:
-                return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage)
-
-            return 1
-
-    ```
-
-    Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation.
-
-!!! Note
-    Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings.
-    This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade
-    is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `exit_reason` in
-    `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when
-    `current_profit < open_relative_stop`.
-
-### *stoploss_from_absolute()*
-
-In some situations it may be confusing to deal with stops relative to current rate. Instead, you may define a stoploss level using an absolute price.
-
-??? Example "Returning a stoploss using absolute price from the custom stoploss function"
-
-    If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short)`.
-
-    ``` python
-
-    from datetime import datetime
-    from freqtrade.persistence import Trade
-    from freqtrade.strategy import IStrategy, stoploss_from_absolute
-
-    class AwesomeStrategy(IStrategy):
-
-        use_custom_stoploss = True
-
-        def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-            dataframe['atr'] = ta.ATR(dataframe, timeperiod=14)
-            return dataframe
-
-        def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                            current_rate: float, current_profit: float, **kwargs) -> float:
-            dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
-            candle = dataframe.iloc[-1].squeeze()
-            return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short)
-
-    ```
-
 ## Additional data (Wallets)
 
 The strategy provides access to the `wallets` object. This contains the current balances on the exchange.

docs/strategy_analysis_example.md

@@ -167,7 +167,7 @@ trades.groupby("pair")["exit_reason"].value_counts()
 # Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)
 
 from freqtrade.configuration import Configuration
-from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats
+from freqtrade.data.btanalysis import load_backtest_stats
 import plotly.express as px
 import pandas as pd
 
@@ -178,20 +178,8 @@ import pandas as pd
 stats = load_backtest_stats(backtest_dir)
 strategy_stats = stats['strategy'][strategy]
 
-dates = []
-profits = []
-for date_profit in strategy_stats['daily_profit']:
-    dates.append(date_profit[0])
-    profits.append(date_profit[1])
-
-equity = 0
-equity_daily = []
-for daily_profit in profits:
-    equity_daily.append(equity)
-    equity += float(daily_profit)
-
-
-df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})
+df = pd.DataFrame(columns=['dates','equity'], data=strategy_stats['daily_profit'])
+df['equity_daily'] = df['equity'].cumsum()
 
 fig = px.line(df, x="dates", y="equity_daily")
 fig.show()

docs/strategy_migration.md

@@ -280,7 +280,7 @@ After:
 
 ``` python hl_lines="3"
 class AwesomeStrategy(IStrategy):
-    def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float,
+    def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float,
                            entry_tag: Optional[str], side: str, **kwargs) -> float:
       return proposed_rate
 ```
@@ -311,12 +311,13 @@ After:
 
 ``` python hl_lines="5 7"
     def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
-                        current_rate: float, current_profit: float, **kwargs) -> float:
+                        current_rate: float, current_profit: float, after_fill: bool, 
+                        **kwargs) -> Optional[float]:
         # once the profit has risen above 10%, keep the stoploss at 7% above the open price
         if current_profit > 0.10:
             return stoploss_from_open(0.07, current_profit, is_short=trade.is_short)
 
-        return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short)
+        return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage)
 
 
 ```

docs/utils.md

@@ -967,7 +967,7 @@ Print trades with id 2 and 3 as json
 freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json
 ```
 
-### Strategy-Updater
+## Strategy-Updater
 
 Updates listed strategies or all strategies within the strategies folder to be v3 compliant.
 If the command runs without --strategy-list then all strategies inside the strategies folder will be converted.

docs/windows_installation.md

@@ -24,15 +24,15 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.8, 3.9, 3.10 and 3.11) and for 64bit Windows.
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.
 
 Other versions must be downloaded from the above link.
 
 ``` powershell
 cd \path\freqtrade
-python -m venv .env
-.env\Scripts\activate.ps1
+python -m venv .venv
+.venv\Scripts\activate.ps1
 # optionally install ta-lib from wheel
 # Eventually adjust the below filename to match the downloaded wheel
 pip install --find-links build_helpers\ TA-Lib -U

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2023.7'
+__version__ = '2023.9'
 
 if 'dev' in __version__:
     from pathlib import Path

freqtrade/__main__.py

@@ -3,7 +3,7 @@
 __main__.py for Freqtrade
 To launch Freqtrade as a module
 
-> python -m freqtrade (with Python >= 3.8)
+> python -m freqtrade (with Python >= 3.9)
 """
 
 from freqtrade import main

freqtrade/commands/__init__.py

@@ -20,7 +20,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_f
                                               start_list_timeframes, start_show_trades)
 from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show,
                                                   start_edge, start_hyperopt,
-                                                  start_lookahead_analysis)
+                                                  start_lookahead_analysis,
+                                                  start_recursive_analysis)
 from freqtrade.commands.pairlist_commands import start_test_pairlist
 from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
 from freqtrade.commands.strategy_utils_commands import start_strategy_update

freqtrade/commands/arguments.py

@@ -122,6 +122,8 @@ ARGS_LOOKAHEAD_ANALYSIS = [
     a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", 'cache')
     ] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"]
 
+ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"]
+
 
 class Arguments:
     """
@@ -206,8 +208,9 @@ class Arguments:
                                         start_list_strategies, start_list_timeframes,
                                         start_lookahead_analysis, start_new_config,
                                         start_new_strategy, start_plot_dataframe, start_plot_profit,
-                                        start_show_trades, start_strategy_update,
-                                        start_test_pairlist, start_trading, start_webserver)
+                                        start_recursive_analysis, start_show_trades,
+                                        start_strategy_update, start_test_pairlist, start_trading,
+                                        start_webserver)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added
@@ -467,3 +470,14 @@ class Arguments:
 
         self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
                          parser=lookahead_analayis_cmd)
+
+        # Add recursive_analysis subcommand
+        recursive_analayis_cmd = subparsers.add_parser(
+            'recursive-analysis',
+            help="Check for potential recursive formula issue.",
+            parents=[_common_parser, _strategy_parser])
+
+        recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
+
+        self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,
+                         parser=recursive_analayis_cmd)

freqtrade/commands/build_config_commands.py

@@ -10,7 +10,7 @@ from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
-from freqtrade.misc import render_template
+from freqtrade.util import render_template
 
 
 logger = logging.getLogger(__name__)
@@ -105,7 +105,7 @@ def ask_user_config() -> Dict[str, Any]:
             "type": "select",
             "name": "exchange_name",
             "message": "Select exchange",
-            "choices": lambda x: [
+            "choices": [
                 "binance",
                 "binanceus",
                 "bittrex",

freqtrade/commands/cli_options.py

@@ -435,13 +435,13 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "dataformat_ohlcv": Arg(
         '--data-format-ohlcv',
-        help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
+        help='Storage format for downloaded candle (OHLCV) data. (default: `feather`).',
         choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "dataformat_trades": Arg(
         '--data-format-trades',
-        help='Storage format for downloaded trades data. (default: `jsongz`).',
-        choices=constants.AVAILABLE_DATAHANDLERS_TRADES,
+        help='Storage format for downloaded trades data. (default: `feather`).',
+        choices=constants.AVAILABLE_DATAHANDLERS,
     ),
     "show_timerange": Arg(
         '--show-timerange',
@@ -705,4 +705,9 @@ AVAILABLE_CLI_OPTIONS = {
         help="Use this csv-filename to store lookahead-analysis-results",
         type=str
     ),
+    "startup_candle": Arg(
+        '--startup-candle',
+        help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).',
+        nargs='+',
+    ),
 }

freqtrade/commands/data_commands.py

@@ -5,12 +5,12 @@ from typing import Any, Dict
 
 from freqtrade.configuration import TimeRange, setup_utils_configuration
 from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config
-from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
-from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main
+from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
+                                      convert_trades_to_ohlcv)
+from freqtrade.data.history import download_data_main
 from freqtrade.enums import RunMode, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
-from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
 from freqtrade.util.binance_mig import migrate_binance_futures_data
 
@@ -53,28 +53,19 @@ def start_convert_trades(args: Dict[str, Any]) -> None:
     # Remove stake-currency to skip checks which are not relevant for datadownload
     config['stake_currency'] = ''
 
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
     if 'timeframes' not in config:
         config['timeframes'] = DL_DATA_TIMEFRAMES
 
     # Init exchange
     exchange = ExchangeResolver.load_exchange(config, validate=False)
     # Manual validations of relevant settings
-    if not config['exchange'].get('skip_pair_validation', False):
-        exchange.validate_pairs(config['pairs'])
-    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
-
-    logger.info(f"About to Convert pairs: {expanded_pairs}, "
-                f"intervals: {config['timeframes']} to {config['datadir']}")
 
     for timeframe in config['timeframes']:
         exchange.validate_timeframes(timeframe)
+
     # Convert downloaded trade data to different timeframes
     convert_trades_to_ohlcv(
-        pairs=expanded_pairs, timeframes=config['timeframes'],
+        pairs=config.get('pairs', []), timeframes=config['timeframes'],
         datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
         data_format_ohlcv=config['dataformat_ohlcv'],
         data_format_trades=config['dataformat_trades'],

freqtrade/commands/deploy_commands.py

@@ -10,7 +10,7 @@ from freqtrade.configuration.directory_operations import copy_sample_files, crea
 from freqtrade.constants import USERPATH_STRATEGIES
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import render_template, render_template_with_fallback
+from freqtrade.util import render_template, render_template_with_fallback
 
 
 logger = logging.getLogger(__name__)
@@ -35,6 +35,10 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
     Deploy new strategy from template to strategy_path
     """
     fallback = 'full'
+    attributes = render_template_with_fallback(
+        templatefile=f"strategy_subtemplates/strategy_attributes_{subtemplate}.j2",
+        templatefallbackfile=f"strategy_subtemplates/strategy_attributes_{fallback}.j2",
+    )
     indicators = render_template_with_fallback(
         templatefile=f"strategy_subtemplates/indicators_{subtemplate}.j2",
         templatefallbackfile=f"strategy_subtemplates/indicators_{fallback}.j2",
@@ -58,6 +62,7 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
 
     strategy_text = render_template(templatefile='base_strategy.py.j2',
                                     arguments={"strategy": strategy_name,
+                                               "attributes": attributes,
                                                "indicators": indicators,
                                                "buy_trend": buy_trend,
                                                "sell_trend": sell_trend,

freqtrade/commands/optimize_commands.py

@@ -144,3 +144,15 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None:
 
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     LookaheadAnalysisSubFunctions.start(config)
+
+
+def start_recursive_analysis(args: Dict[str, Any]) -> None:
+    """
+    Start the backtest recursive tester script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+    RecursiveAnalysisSubFunctions.start(config)

freqtrade/commands/webserver_commands.py

@@ -7,9 +7,10 @@ def start_webserver(args: Dict[str, Any]) -> None:
     """
     Main entry point for webserver mode
     """
-    from freqtrade.configuration import Configuration
+    from freqtrade.configuration import setup_utils_configuration
     from freqtrade.rpc.api_server import ApiServer
 
     # Initialize configuration
-    config = Configuration(args, RunMode.WEBSERVER).get_config()
+
+    config = setup_utils_configuration(args, RunMode.WEBSERVER)
     ApiServer(config, standalone=True)

freqtrade/configuration/__init__.py

@@ -3,4 +3,5 @@
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.detect_environment import running_in_docker
 from freqtrade.configuration.timerange import TimeRange

freqtrade/configuration/config_validation.py

@@ -51,6 +51,8 @@ def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> D
             conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
         else:
             conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
+    elif conf.get('runmode', RunMode.OTHER) == RunMode.WEBSERVER:
+        conf_schema['required'] = constants.SCHEMA_MINIMAL_WEBSERVER
     else:
         conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED
     try:

freqtrade/configuration/configuration.py

@@ -490,6 +490,9 @@ class Configuration:
         self._args_to_config(config, argname='lookahead_analysis_exportfilename',
                              logstring='Path to store lookahead-analysis-results: {}')
 
+        self._args_to_config(config, argname='startup_candle',
+                             logstring='Startup candle to be used on recursive analysis: {}')
+
     def _process_runmode(self, config: Config) -> None:
 
         self._args_to_config(config, argname='dry_run',

freqtrade/configuration/environment_vars.py

@@ -41,7 +41,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
             key = env_var.replace(prefix, '')
             for k in reversed(key.split('__')):
                 val = {k.lower(): get_var_typed(val)
-                       if type(val) != dict and k not in no_convert else val}
+                       if not isinstance(val, dict) and k not in no_convert else val}
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
 

freqtrade/constants.py

@@ -33,13 +33,12 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
                          'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
                          'ProfitDrawDownHyperOptLoss']
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
-                       'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
+                       'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter',
                        'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
                        'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod',
                          'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
-AVAILABLE_DATAHANDLERS_TRADES = ['json', 'jsongz', 'hdf5', 'feather']
-AVAILABLE_DATAHANDLERS = AVAILABLE_DATAHANDLERS_TRADES + ['parquet']
+AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet']
 BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
 BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
 BACKTEST_CACHE_DEFAULT = 'day'
@@ -50,6 +49,15 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
 # Don't modify sequence of DEFAULT_TRADES_COLUMNS
 # it has wide consequences for stored trades files
 DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
+TRADES_DTYPES = {
+    'timestamp': 'int64',
+    'id': 'str',
+    'type': 'str',
+    'side': 'str',
+    'price': 'float64',
+    'amount': 'float64',
+    'cost': 'float64',
+}
 TRADING_MODES = ['spot', 'margin', 'futures']
 MARGIN_MODES = ['cross', 'isolated', '']
 
@@ -69,7 +77,8 @@ DL_DATA_TIMEFRAMES = ['1m', '5m']
 
 ENV_VAR_PREFIX = 'FREQTRADE__'
 
-NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
+CANCELED_EXCHANGE_STATES = ('cancelled', 'canceled', 'expired')
+NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ('closed',)
 
 # Define decimals per coin for outputs
 # Only used for outputs.
@@ -153,7 +162,7 @@ CONF_SCHEMA = {
             },
         },
         'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5},
-        'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True, 'minimum': -1},
+        'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
         'trailing_stop': {'type': 'boolean'},
         'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
         'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
@@ -169,6 +178,11 @@ CONF_SCHEMA = {
         'minimum_trade_amount': {'type': 'number', 'default': 10},
         'targeted_trade_amount': {'type': 'number', 'default': 20},
         'lookahead_analysis_exportfilename': {'type': 'string'},
+        'startup_candle': {
+            'type': 'array',
+            'uniqueItems': True,
+            'default': [199, 399, 499, 999, 1999],
+        },
         'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99},
         'backtest_breakdown': {
             'type': 'array',
@@ -446,12 +460,12 @@ CONF_SCHEMA = {
         'dataformat_ohlcv': {
             'type': 'string',
             'enum': AVAILABLE_DATAHANDLERS,
-            'default': 'json'
+            'default': 'feather'
         },
         'dataformat_trades': {
             'type': 'string',
-            'enum': AVAILABLE_DATAHANDLERS_TRADES,
-            'default': 'jsongz'
+            'enum': AVAILABLE_DATAHANDLERS,
+            'default': 'feather'
         },
         'position_adjustment_enable': {'type': 'boolean'},
         'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1},
@@ -461,7 +475,6 @@ CONF_SCHEMA = {
             'type': 'object',
             'properties': {
                 'name': {'type': 'string'},
-                'sandbox': {'type': 'boolean', 'default': False},
                 'key': {'type': 'string', 'default': ''},
                 'secret': {'type': 'string', 'default': ''},
                 'password': {'type': 'string', 'default': ''},
@@ -668,6 +681,9 @@ SCHEMA_MINIMAL_REQUIRED = [
     'dataformat_ohlcv',
     'dataformat_trades',
 ]
+SCHEMA_MINIMAL_WEBSERVER = SCHEMA_MINIMAL_REQUIRED + [
+    'api_server',
+]
 
 CANCEL_REASON = {
     "TIMEOUT": "cancelled due to timeout",
@@ -678,6 +694,7 @@ CANCEL_REASON = {
     "CANCELLED_ON_EXCHANGE": "cancelled on exchange",
     "FORCE_EXIT": "forcesold",
     "REPLACE": "cancelled to be replaced by new limit order",
+    "REPLACE_FAILED": "failed to replace order, deleting Trade",
     "USER_CANCEL": "user requested order cancel"
 }
 
@@ -699,3 +716,6 @@ Config = Dict[str, Any]
 # Exchange part of the configuration.
 ExchangeConfig = Dict[str, Any]
 IntOrInf = float
+
+
+EntryExecuteMode = Literal['initial', 'pos_adjust', 'replace']

freqtrade/data/btanalysis.py

@@ -5,16 +5,17 @@ import logging
 from copy import copy
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 import numpy as np
 import pandas as pd
 
 from freqtrade.constants import LAST_BT_RESULT_FN, IntOrInf
 from freqtrade.exceptions import OperationalException
-from freqtrade.misc import json_load
+from freqtrade.misc import file_dump_json, json_load
 from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
 from freqtrade.persistence import LocalTrade, Trade, init_db
+from freqtrade.types import BacktestHistoryEntryType, BacktestResultType
 
 
 logger = logging.getLogger(__name__)
@@ -128,7 +129,7 @@ def load_backtest_metadata(filename: Union[Path, str]) -> Dict[str, Any]:
         raise OperationalException('Unexpected error while loading backtest metadata.') from e
 
 
-def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
+def load_backtest_stats(filename: Union[Path, str]) -> BacktestResultType:
     """
     Load backtest statistics file.
     :param filename: pathlib.Path object, or string pointing to the file.
@@ -147,21 +148,21 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
     # Legacy list format does not contain metadata.
     if isinstance(data, dict):
         data['metadata'] = load_backtest_metadata(filename)
-
     return data
 
 
 def load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
     """
-    Load one strategy from multi-strategy result
-    and merge it with results
+    Load one strategy from multi-strategy result and merge it with results
     :param strategy_name: Name of the strategy contained in the result
     :param filename: Backtest-result-filename to load
     :param results: dict to merge the result to.
     """
     bt_data = load_backtest_stats(filename)
-    for k in ('metadata', 'strategy'):
+    k: Literal['metadata', 'strategy']
+    for k in ('metadata', 'strategy'):  # type: ignore
         results[k][strategy_name] = bt_data[k][strategy_name]
+    results['metadata'][strategy_name]['filename'] = filename.stem
     comparison = bt_data['strategy_comparison']
     for i in range(len(comparison)):
         if comparison[i]['key'] == strategy_name:
@@ -174,24 +175,37 @@ def _get_backtest_files(dirname: Path) -> List[Path]:
     return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
 
 
-def get_backtest_resultlist(dirname: Path):
+def get_backtest_result(filename: Path) -> List[BacktestHistoryEntryType]:
+    """
+    Get backtest result read from metadata file
+    """
+    return [
+        {
+            'filename': filename.stem,
+            'strategy': s,
+            'notes': v.get('notes', ''),
+            'run_id': v['run_id'],
+            'backtest_start_time': v['backtest_start_time'],
+        } for s, v in load_backtest_metadata(filename).items()
+    ]
+
+
+def get_backtest_resultlist(dirname: Path) -> List[BacktestHistoryEntryType]:
     """
     Get list of backtest results read from metadata files
     """
-    results = []
-    for filename in _get_backtest_files(dirname):
-        metadata = load_backtest_metadata(filename)
-        if not metadata:
-            continue
-        for s, v in metadata.items():
-            results.append({
-                'filename': filename.stem,
-                'strategy': s,
-                'run_id': v['run_id'],
-                'backtest_start_time': v['backtest_start_time'],
-
-            })
-    return results
+    return [
+        {
+            'filename': filename.stem,
+            'strategy': s,
+            'run_id': v['run_id'],
+            'notes': v.get('notes', ''),
+            'backtest_start_time': v['backtest_start_time'],
+        }
+        for filename in _get_backtest_files(dirname)
+        for s, v in load_backtest_metadata(filename).items()
+        if v
+    ]
 
 
 def delete_backtest_result(file_abs: Path):
@@ -205,6 +219,21 @@ def delete_backtest_result(file_abs: Path):
     file_abs_meta.unlink()
 
 
+def update_backtest_metadata(filename: Path, strategy: str, content: Dict[str, Any]):
+    """
+    Updates backtest metadata file with new content.
+    :raises: ValueError if metadata file does not exist, or strategy is not in this file.
+    """
+    metadata = load_backtest_metadata(filename)
+    if not metadata:
+        raise ValueError("File does not exist.")
+    if strategy not in metadata:
+        raise ValueError("Strategy not in metadata.")
+    metadata[strategy].update(content)
+    # Write data again.
+    file_dump_json(get_backtest_metadata_filename(filename), metadata)
+
+
 def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
                                  min_backtest_date: Optional[datetime] = None) -> Dict[str, Any]:
     """

freqtrade/data/converter/__init__.py

@@ -0,0 +1,28 @@
+from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format,
+                                                ohlcv_fill_up_missing_data, ohlcv_to_dataframe,
+                                                order_book_to_dataframe, reduce_dataframe_footprint,
+                                                trim_dataframe, trim_dataframes)
+from freqtrade.data.converter.trade_converter import (convert_trades_format,
+                                                      convert_trades_to_ohlcv, trades_convert_types,
+                                                      trades_df_remove_duplicates,
+                                                      trades_dict_to_list, trades_list_to_df,
+                                                      trades_to_ohlcv)
+
+
+__all__ = [
+    'clean_ohlcv_dataframe',
+    'convert_ohlcv_format',
+    'ohlcv_fill_up_missing_data',
+    'ohlcv_to_dataframe',
+    'order_book_to_dataframe',
+    'reduce_dataframe_footprint',
+    'trim_dataframe',
+    'trim_dataframes',
+    'convert_trades_format',
+    'convert_trades_to_ohlcv',
+    'trades_convert_types',
+    'trades_df_remove_duplicates',
+    'trades_dict_to_list',
+    'trades_list_to_df',
+    'trades_to_ohlcv',
+]

freqtrade/data/converter/converter.py

@@ -1,16 +1,14 @@
 """
 Functions to convert data from one format to another
 """
-import itertools
 import logging
-from operator import itemgetter
-from typing import Dict, List
+from typing import Dict
 
 import numpy as np
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, Config
 from freqtrade.enums import CandleType, TradingMode
 
 
@@ -106,7 +104,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict)
 
     # Forwardfill close for missing columns
-    df['close'] = df['close'].fillna(method='ffill')
+    df['close'] = df['close'].ffill()
     # Use close for "open, high, low"
     df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna(
         value={'open': df['close'],
@@ -195,76 +193,6 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     return frame
 
 
-def trades_remove_duplicates(trades: List[List]) -> List[List]:
-    """
-    Removes duplicates from the trades list.
-    Uses itertools.groupby to avoid converting to pandas.
-    Tests show it as being pretty efficient on lists of 4M Lists.
-    :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns
-    :return: same format as above, but with duplicates removed
-    """
-    return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))]
-
-
-def trades_dict_to_list(trades: List[Dict]) -> TradeList:
-    """
-    Convert fetch_trades result into a List (to be more memory efficient).
-    :param trades: List of trades, as returned by ccxt.fetch_trades.
-    :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
-    """
-    return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
-
-
-def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame:
-    """
-    Converts trades list to OHLCV list
-    :param trades: List of trades, as returned by ccxt.fetch_trades.
-    :param timeframe: Timeframe to resample data to
-    :return: OHLCV Dataframe.
-    :raises: ValueError if no trades are provided
-    """
-    from freqtrade.exchange import timeframe_to_minutes
-    timeframe_minutes = timeframe_to_minutes(timeframe)
-    if not trades:
-        raise ValueError('Trade-list empty.')
-    df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
-    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms',
-                                     utc=True,)
-    df = df.set_index('timestamp')
-
-    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
-    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
-    df_new['date'] = df_new.index
-    # Drop 0 volume rows
-    df_new = df_new.dropna()
-    return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS]
-
-
-def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
-    """
-    Convert trades from one format to another format.
-    :param config: Config dictionary
-    :param convert_from: Source format
-    :param convert_to: Target format
-    :param erase: Erase source data (does not apply if source and target format are identical)
-    """
-    from freqtrade.data.history.idatahandler import get_datahandler
-    src = get_datahandler(config['datadir'], convert_from)
-    trg = get_datahandler(config['datadir'], convert_to)
-
-    if 'pairs' not in config:
-        config['pairs'] = src.trades_get_pairs(config['datadir'])
-    logger.info(f"Converting trades for {config['pairs']}")
-
-    for pair in config['pairs']:
-        data = src.trades_load(pair=pair)
-        logger.info(f"Converting {len(data)} trades for {pair}")
-        trg.trades_store(pair, data)
-        if erase and convert_from != convert_to:
-            logger.info(f"Deleting source Trade data for {pair}.")
-            src.trades_purge(pair=pair)
-
-
 def convert_ohlcv_format(
     config: Config,
     convert_from: str,

freqtrade/data/converter/trade_converter.py

@@ -0,0 +1,144 @@
+"""
+Functions to convert data from one format to another
+"""
+import logging
+from pathlib import Path
+from typing import Dict, List
+
+import pandas as pd
+from pandas import DataFrame, to_datetime
+
+from freqtrade.configuration import TimeRange
+from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
+                                 Config, TradeList)
+from freqtrade.enums import CandleType
+
+
+logger = logging.getLogger(__name__)
+
+
+def trades_df_remove_duplicates(trades: pd.DataFrame) -> pd.DataFrame:
+    """
+    Removes duplicates from the trades DataFrame.
+    Uses pandas.DataFrame.drop_duplicates to remove duplicates based on the 'timestamp' column.
+    :param trades: DataFrame with the columns constants.DEFAULT_TRADES_COLUMNS
+    :return: DataFrame with duplicates removed based on the 'timestamp' column
+    """
+    return trades.drop_duplicates(subset=['timestamp', 'id'])
+
+
+def trades_dict_to_list(trades: List[Dict]) -> TradeList:
+    """
+    Convert fetch_trades result into a List (to be more memory efficient).
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
+    """
+    return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades]
+
+
+def trades_convert_types(trades: DataFrame) -> DataFrame:
+    """
+    Convert Trades dtypes and add 'date' column
+    """
+    trades = trades.astype(TRADES_DTYPES)
+    trades['date'] = to_datetime(trades['timestamp'], unit='ms', utc=True)
+    return trades
+
+
+def trades_list_to_df(trades: TradeList, convert: bool = True):
+    """
+    convert trades list to dataframe
+    :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns
+    """
+    if not trades:
+        df = DataFrame(columns=DEFAULT_TRADES_COLUMNS)
+    else:
+        df = DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
+
+    if convert:
+        df = trades_convert_types(df)
+
+    return df
+
+
+def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame:
+    """
+    Converts trades list to OHLCV list
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :param timeframe: Timeframe to resample data to
+    :return: OHLCV Dataframe.
+    :raises: ValueError if no trades are provided
+    """
+    from freqtrade.exchange import timeframe_to_minutes
+    timeframe_minutes = timeframe_to_minutes(timeframe)
+    if trades.empty:
+        raise ValueError('Trade-list empty.')
+    df = trades.set_index('date', drop=True)
+
+    df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
+    df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
+    df_new['date'] = df_new.index
+    # Drop 0 volume rows
+    df_new = df_new.dropna()
+    return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS]
+
+
+def convert_trades_to_ohlcv(
+    pairs: List[str],
+    timeframes: List[str],
+    datadir: Path,
+    timerange: TimeRange,
+    erase: bool = False,
+    data_format_ohlcv: str = 'feather',
+    data_format_trades: str = 'feather',
+    candle_type: CandleType = CandleType.SPOT
+) -> None:
+    """
+    Convert stored trades data to ohlcv data
+    """
+    from freqtrade.data.history.idatahandler import get_datahandler
+    data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
+    data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
+    if not pairs:
+        pairs = data_handler_trades.trades_get_pairs(datadir)
+
+    logger.info(f"About to convert pairs: '{', '.join(pairs)}', "
+                f"intervals: '{', '.join(timeframes)}' to {datadir}")
+
+    for pair in pairs:
+        trades = data_handler_trades.trades_load(pair)
+        for timeframe in timeframes:
+            if erase:
+                if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
+                    logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
+            try:
+                ohlcv = trades_to_ohlcv(trades, timeframe)
+                # Store ohlcv
+                data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
+            except ValueError:
+                logger.exception(f'Could not convert {pair} to OHLCV.')
+
+
+def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
+    """
+    Convert trades from one format to another format.
+    :param config: Config dictionary
+    :param convert_from: Source format
+    :param convert_to: Target format
+    :param erase: Erase source data (does not apply if source and target format are identical)
+    """
+    from freqtrade.data.history.idatahandler import get_datahandler
+    src = get_datahandler(config['datadir'], convert_from)
+    trg = get_datahandler(config['datadir'], convert_to)
+
+    if 'pairs' not in config:
+        config['pairs'] = src.trades_get_pairs(config['datadir'])
+    logger.info(f"Converting trades for {config['pairs']}")
+
+    for pair in config['pairs']:
+        data = src.trades_load(pair=pair)
+        logger.info(f"Converting {len(data)} trades for {pair}")
+        trg.trades_store(pair, data)
+        if erase and convert_from != convert_to:
+            logger.info(f"Deleting source Trade data for {pair}.")
+            src.trades_purge(pair=pair)

freqtrade/data/dataprovider.py

@@ -17,7 +17,7 @@ from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWith
 from freqtrade.data.history import load_pair_history
 from freqtrade.enums import CandleType, RPCMessageType, RunMode
 from freqtrade.exceptions import ExchangeError, OperationalException
-from freqtrade.exchange import Exchange, timeframe_to_seconds
+from freqtrade.exchange import Exchange, timeframe_to_prev_date, timeframe_to_seconds
 from freqtrade.exchange.types import OrderBook
 from freqtrade.misc import append_candles_to_dataframe
 from freqtrade.rpc import RPCManager
@@ -46,6 +46,8 @@ class DataProvider:
         self.__rpc = rpc
         self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
         self.__slice_index: Optional[int] = None
+        self.__slice_date: Optional[datetime] = None
+
         self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
         self.__producer_pairs_df: Dict[str,
                                        Dict[PairWithTimeframe, Tuple[DataFrame, datetime]]] = {}
@@ -64,10 +66,19 @@ class DataProvider:
     def _set_dataframe_max_index(self, limit_index: int):
         """
         Limit analyzed dataframe to max specified index.
+        Only relevant in backtesting.
         :param limit_index: dataframe index.
         """
         self.__slice_index = limit_index
 
+    def _set_dataframe_max_date(self, limit_date: datetime):
+        """
+        Limit infomrative dataframe to max specified index.
+        Only relevant in backtesting.
+        :param limit_date: "current date"
+        """
+        self.__slice_date = limit_date
+
     def _set_cached_df(
         self,
         pair: str,
@@ -284,7 +295,7 @@ class DataProvider:
     def historic_ohlcv(
         self,
         pair: str,
-        timeframe: Optional[str] = None,
+        timeframe: str,
         candle_type: str = ''
     ) -> DataFrame:
         """
@@ -307,10 +318,10 @@ class DataProvider:
             timerange.subtract_start(tf_seconds * startup_candles)
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
                 pair=pair,
-                timeframe=timeframe or self._config['timeframe'],
+                timeframe=timeframe,
                 datadir=self._config['datadir'],
                 timerange=timerange,
-                data_format=self._config.get('dataformat_ohlcv', 'json'),
+                data_format=self._config['dataformat_ohlcv'],
                 candle_type=_candle_type,
 
             )
@@ -354,7 +365,13 @@ class DataProvider:
             data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
         else:
             # Get historical OHLCV data (cached on disk).
+            timeframe = timeframe or self._config['timeframe']
             data = self.historic_ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
+            # Cut date to timeframe-specific date.
+            # This is necessary to prevent lookahead bias in callbacks through informative pairs.
+            if self.__slice_date:
+                cutoff_date = timeframe_to_prev_date(timeframe, self.__slice_date)
+                data = data.loc[data['date'] < cutoff_date]
         if len(data) == 0:
             logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).")
         return data

freqtrade/data/entryexitanalysis.py

@@ -119,8 +119,15 @@ def _do_group_table_output(bigdf, glist, csv_path: Path, to_csv=False, ):
             new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0)
             new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0)
 
-            new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss',
-                           'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss']
+            new['exp_ratio'] = (
+                (
+                    (1 + (new['avg_win'] / abs(new['avg_loss']))) * (new['wl_ratio_pct'] / 100)
+                ) - 1).fillna(0)
+
+            new.columns = ['total_num_buys', 'wins', 'losses',
+                           'profit_abs_wins', 'profit_abs_loss',
+                           'profit_tot', 'wl_ratio_pct',
+                           'avg_win', 'avg_loss', 'exp_ratio']
 
             sortcols = ['total_num_buys']
 
@@ -204,6 +211,7 @@ def prepare_results(analysed_trades, stratname,
                     timerange=None):
     res_df = pd.DataFrame()
     for pair, trades in analysed_trades[stratname].items():
+        trades.dropna(subset=['close_date'], inplace=True)
         res_df = pd.concat([res_df, trades], ignore_index=True)
 
     res_df = _select_rows_within_dates(res_df, timerange)

freqtrade/data/history/featherdatahandler.py

@@ -4,7 +4,7 @@ from typing import Optional
 from pandas import DataFrame, read_feather, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
 from freqtrade.enums import CandleType
 
 from .idatahandler import IDataHandler
@@ -82,43 +82,41 @@ class FeatherDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def trades_store(self, pair: str, data: TradeList) -> None:
+    def _trades_store(self, pair: str, data: DataFrame) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         self.create_dir_if_needed(filename)
+        data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4')
 
-        tradesdata = DataFrame(data, columns=DEFAULT_TRADES_COLUMNS)
-        tradesdata.to_feather(filename, compression_level=9, compression='lz4')
-
-    def trades_append(self, pair: str, data: TradeList):
+    def trades_append(self, pair: str, data: DataFrame):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         if not filename.exists():
-            return []
+            return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 
         tradesdata = read_feather(filename)
 
-        return tradesdata.values.tolist()
+        return tradesdata
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/data/history/hdf5datahandler.py

@@ -5,7 +5,7 @@ import numpy as np
 import pandas as pd
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
 from freqtrade.enums import CandleType
 
 from .idatahandler import IDataHandler
@@ -100,42 +100,42 @@ class HDF5DataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def trades_store(self, pair: str, data: TradeList) -> None:
+    def _trades_store(self, pair: str, data: pd.DataFrame) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         key = self._pair_trades_key(pair)
 
-        pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf(
+        data.to_hdf(
             self._pair_trades_filename(self._datadir, pair), key,
             mode='a', complevel=9, complib='blosc',
             format='table', data_columns=['timestamp']
         )
 
-    def trades_append(self, pair: str, data: TradeList):
+    def trades_append(self, pair: str, data: pd.DataFrame):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
         """
         Load a pair from h5 file.
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         key = self._pair_trades_key(pair)
         filename = self._pair_trades_filename(self._datadir, pair)
 
         if not filename.exists():
-            return []
+            return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)
         where = []
         if timerange:
             if timerange.starttype == 'date':
@@ -145,7 +145,7 @@ class HDF5DataHandler(IDataHandler):
 
         trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)
         trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
-        return trades.values.tolist()
+        return trades
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/data/history/history_utils.py

@@ -9,15 +9,17 @@ from pandas import DataFrame, concat
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS,
                                  DL_DATA_TIMEFRAMES, Config)
-from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe,
-                                      trades_remove_duplicates, trades_to_ohlcv)
+from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv,
+                                      ohlcv_to_dataframe, trades_df_remove_duplicates,
+                                      trades_list_to_df)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
 from freqtrade.enums import CandleType
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
-from freqtrade.util import format_ms_time
+from freqtrade.util import dt_ts, format_ms_time
 from freqtrade.util.binance_mig import migrate_binance_futures_data
+from freqtrade.util.datetime_helpers import dt_now
 
 
 logger = logging.getLogger(__name__)
@@ -69,7 +71,7 @@ def load_data(datadir: Path,
               fill_up_missing: bool = True,
               startup_candles: int = 0,
               fail_without_data: bool = False,
-              data_format: str = 'json',
+              data_format: str = 'feather',
               candle_type: CandleType = CandleType.SPOT,
               user_futures_funding_rate: Optional[int] = None,
               ) -> Dict[str, DataFrame]:
@@ -349,24 +351,27 @@ def _download_trades_history(exchange: Exchange,
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
         # DEFAULT_TRADES_COLUMNS: 1 -> id
 
-        if trades and since < trades[0][0]:
+        if not trades.empty and since > 0 and since < trades.iloc[0]['timestamp']:
             # since is before the first trade
-            logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
-            trades = []
+            logger.info(f"Start ({trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}) earlier than "
+                        f"available data. Redownloading trades for {pair}...")
+            trades = trades_list_to_df([])
 
-        if not since:
-            since = int((datetime.now() - timedelta(days=new_pairs_days)).timestamp()) * 1000
-
-        from_id = trades[-1][1] if trades else None
-        if trades and since < trades[-1][0]:
+        from_id = trades.iloc[-1]['id'] if not trades.empty else None
+        if not trades.empty and since < trades.iloc[-1]['timestamp']:
             # Reset since to the last available point
             # - 5 seconds (to ensure we're getting all trades)
-            since = trades[-1][0] - (5 * 1000)
+            since = trades.iloc[-1]['timestamp'] - (5 * 1000)
             logger.info(f"Using last trade date -5s - Downloading trades for {pair} "
                         f"since: {format_ms_time(since)}.")
 
-        logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}")
-        logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}")
+        if not since:
+            since = dt_ts(dt_now() - timedelta(days=new_pairs_days))
+
+        logger.debug("Current Start: %s", 'None' if trades.empty else
+                     f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
+        logger.debug("Current End: %s", 'None' if trades.empty else
+                     f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}")
         logger.info(f"Current Amount of trades: {len(trades)}")
 
         # Default since_ms to 30 days if nothing is given
@@ -375,13 +380,16 @@ def _download_trades_history(exchange: Exchange,
                                                   until=until,
                                                   from_id=from_id,
                                                   )
-        trades.extend(new_trades[1])
+        new_trades_df = trades_list_to_df(new_trades[1])
+        trades = concat([trades, new_trades_df], axis=0)
         # Remove duplicates to make sure we're not storing data we don't need
-        trades = trades_remove_duplicates(trades)
+        trades = trades_df_remove_duplicates(trades)
         data_handler.trades_store(pair, data=trades)
 
-        logger.debug(f"New Start: {format_ms_time(trades[0][0])}")
-        logger.debug(f"New End: {format_ms_time(trades[-1][0])}")
+        logger.debug("New Start: %s", 'None' if trades.empty else
+                     f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
+        logger.debug("New End: %s", 'None' if trades.empty else
+                     f"{trades.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}")
         logger.info(f"New Amount of trades: {len(trades)}")
         return True
 
@@ -394,7 +402,7 @@ def _download_trades_history(exchange: Exchange,
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
                                  timerange: TimeRange, new_pairs_days: int = 30,
-                                 erase: bool = False, data_format: str = 'jsongz') -> List[str]:
+                                 erase: bool = False, data_format: str = 'feather') -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -421,36 +429,6 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
     return pairs_not_available
 
 
-def convert_trades_to_ohlcv(
-    pairs: List[str],
-    timeframes: List[str],
-    datadir: Path,
-    timerange: TimeRange,
-    erase: bool = False,
-    data_format_ohlcv: str = 'json',
-    data_format_trades: str = 'jsongz',
-    candle_type: CandleType = CandleType.SPOT
-) -> None:
-    """
-    Convert stored trades data to ohlcv data
-    """
-    data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
-    data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
-
-    for pair in pairs:
-        trades = data_handler_trades.trades_load(pair)
-        for timeframe in timeframes:
-            if erase:
-                if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
-                    logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
-            try:
-                ohlcv = trades_to_ohlcv(trades, timeframe)
-                # Store ohlcv
-                data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
-            except ValueError:
-                logger.exception(f'Could not convert {pair} to OHLCV.')
-
-
 def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
     """
     Get the maximum common timerange for the given backtest data.

freqtrade/data/history/idatahandler.py

@@ -15,8 +15,9 @@ from pandas import DataFrame
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import ListPairsWithTimeframes, TradeList
-from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe
+from freqtrade.constants import DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes
+from freqtrade.data.converter import (clean_ohlcv_dataframe, trades_convert_types,
+                                      trades_df_remove_duplicates, trim_dataframe)
 from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exchange import timeframe_to_seconds
 
@@ -170,32 +171,42 @@ class IDataHandler(ABC):
         return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
 
     @abstractmethod
-    def trades_store(self, pair: str, data: TradeList) -> None:
+    def _trades_store(self, pair: str, data: DataFrame) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
 
     @abstractmethod
-    def trades_append(self, pair: str, data: TradeList):
+    def trades_append(self, pair: str, data: DataFrame):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
 
     @abstractmethod
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
 
+    def trades_store(self, pair: str, data: DataFrame) -> None:
+        """
+        Store trades data (list of Dicts) to file
+        :param pair: Pair - used for filename
+        :param data: Dataframe containing trades
+                     column sequence as in DEFAULT_TRADES_COLUMNS
+        """
+        # Filter on expected columns (will remove the actual date column).
+        self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS])
+
     def trades_purge(self, pair: str) -> bool:
         """
         Remove data for this pair
@@ -208,7 +219,7 @@ class IDataHandler(ABC):
             return True
         return False
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         Removes duplicates in the process.
@@ -216,7 +227,10 @@ class IDataHandler(ABC):
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+        trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
+
+        trades = trades_convert_types(trades)
+        return trades
 
     @classmethod
     def create_dir_if_needed(cls, datadir: Path):
@@ -427,6 +441,6 @@ def get_datahandler(datadir: Path, data_format: Optional[str] = None,
     """
 
     if not data_handler:
-        HandlerClass = get_datahandlerclass(data_format or 'json')
+        HandlerClass = get_datahandlerclass(data_format or 'feather')
         data_handler = HandlerClass(datadir)
     return data_handler

freqtrade/data/history/jsondatahandler.py

@@ -6,8 +6,8 @@ from pandas import DataFrame, read_json, to_datetime
 
 from freqtrade import misc
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList
-from freqtrade.data.converter import trades_dict_to_list
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
+from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df
 from freqtrade.enums import CandleType
 
 from .idatahandler import IDataHandler
@@ -94,45 +94,46 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def trades_store(self, pair: str, data: TradeList) -> None:
+    def _trades_store(self, pair: str, data: DataFrame) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         filename = self._pair_trades_filename(self._datadir, pair)
-        misc.file_dump_json(filename, data, is_zip=self._use_zip)
+        trades = data.values.tolist()
+        misc.file_dump_json(filename, trades, is_zip=self._use_zip)
 
-    def trades_append(self, pair: str, data: TradeList):
+    def trades_append(self, pair: str, data: DataFrame):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         raise NotImplementedError()
 
-    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...
         :param pair: Load trades for this pair
         :param timerange: Timerange to load trades for - currently not implemented
-        :return: List of trades
+        :return: Dataframe containing trades
         """
         filename = self._pair_trades_filename(self._datadir, pair)
         tradesdata = misc.file_load_json(filename)
 
         if not tradesdata:
-            return []
+            return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 
         if isinstance(tradesdata[0], dict):
             # Convert trades dict to list
             logger.info("Old trades format detected - converting")
             tradesdata = trades_dict_to_list(tradesdata)
             pass
-        return tradesdata
+        return trades_list_to_df(tradesdata, convert=False)
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/data/history/parquetdatahandler.py

@@ -4,7 +4,7 @@ from typing import Optional
 from pandas import DataFrame, read_parquet, to_datetime
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
 from freqtrade.enums import CandleType
 
 from .idatahandler import IDataHandler
@@ -81,25 +81,22 @@ class ParquetDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def trades_store(self, pair: str, data: TradeList) -> None:
+    def _trades_store(self, pair: str, data: DataFrame) -> None:
         """
         Store trades data (list of Dicts) to file
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
-        # filename = self._pair_trades_filename(self._datadir, pair)
+        filename = self._pair_trades_filename(self._datadir, pair)
+        self.create_dir_if_needed(filename)
+        data.reset_index(drop=True).to_parquet(filename)
 
-        raise NotImplementedError()
-        # array = pa.array(data)
-        # array
-        # feather.write_feather(data, filename)
-
-    def trades_append(self, pair: str, data: TradeList):
+    def trades_append(self, pair: str, data: DataFrame):
         """
         Append data to existing files
         :param pair: Pair - used for filename
-        :param data: List of Lists containing trade data,
+        :param data: Dataframe containing trades
                      column sequence as in DEFAULT_TRADES_COLUMNS
         """
         raise NotImplementedError()
@@ -112,14 +109,13 @@ class ParquetDataHandler(IDataHandler):
         :param timerange: Timerange to load trades for - currently not implemented
         :return: List of trades
         """
-        raise NotImplementedError()
-        # filename = self._pair_trades_filename(self._datadir, pair)
-        # tradesdata = misc.file_load_json(filename)
+        filename = self._pair_trades_filename(self._datadir, pair)
+        if not filename.exists():
+            return DataFrame(columns=DEFAULT_TRADES_COLUMNS)
 
-        # if not tradesdata:
-        #     return []
+        tradesdata = read_parquet(filename)
 
-        # return tradesdata
+        return tradesdata
 
     @classmethod
     def _get_file_extension(cls):

freqtrade/edge/edge_positioning.py

@@ -115,7 +115,7 @@ class Edge:
                 exchange=self.exchange,
                 timeframe=self.strategy.timeframe,
                 timerange=timerange_startup,
-                data_format=self.config.get('dataformat_ohlcv', 'json'),
+                data_format=self.config['dataformat_ohlcv'],
                 candle_type=self.config.get('candle_type_def', CandleType.SPOT),
             )
             # Download informative pairs too
@@ -132,7 +132,7 @@ class Edge:
                     exchange=self.exchange,
                     timeframe=timeframe,
                     timerange=timerange_startup,
-                    data_format=self.config.get('dataformat_ohlcv', 'json'),
+                    data_format=self.config['dataformat_ohlcv'],
                     candle_type=self.config.get('candle_type_def', CandleType.SPOT),
                 )
 
@@ -142,7 +142,7 @@ class Edge:
             timeframe=self.strategy.timeframe,
             timerange=self._timerange,
             startup_candles=self.strategy.startup_candle_count,
-            data_format=self.config.get('dataformat_ohlcv', 'json'),
+            data_format=self.config['dataformat_ohlcv'],
             candle_type=self.config.get('candle_type_def', CandleType.SPOT),
         )
 

freqtrade/exchange/binance.py

@@ -21,6 +21,8 @@ class Binance(Exchange):
 
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
+        "stop_price_param": "stopPrice",
+        "stop_price_prop": "stopPrice",
         "stoploss_order_types": {"limit": "stop_loss_limit"},
         "order_time_in_force": ["GTC", "FOK", "IOC", "PO"],
         "ohlcv_candle_limit": 1000,

freqtrade/exchange/bybit.py

@@ -1,16 +1,16 @@
 """ Bybit exchange subclass """
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional, Tuple
 
 import ccxt
 
 from freqtrade.constants import BuySell
-from freqtrade.enums import MarginMode, PriceType, TradingMode
+from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode
 from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
 from freqtrade.exchange import Exchange
 from freqtrade.exchange.common import retrier
-from freqtrade.exchange.exchange_utils import timeframe_to_msecs
+from freqtrade.util.datetime_helpers import dt_now, dt_ts
 
 
 logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class Bybit(Exchange):
     """
 
     _ft_has: Dict = {
-        "ohlcv_candle_limit": 200,
+        "ohlcv_candle_limit": 1000,
         "ohlcv_has_history": True,
     }
     _ft_has_futures: Dict = {
@@ -36,6 +36,8 @@ class Bybit(Exchange):
         "funding_fee_timeframe": "8h",
         "stoploss_on_exchange": True,
         "stoploss_order_types": {"limit": "limit", "market": "market"},
+        # bybit response parsing fails to populate stopLossPrice
+        "stop_price_prop": "stopPrice",
         "stop_price_type_field": "triggerBy",
         "stop_price_type_value_mapping": {
             PriceType.LAST: "LastPrice",
@@ -91,28 +93,13 @@ class Bybit(Exchange):
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    async def _fetch_funding_rate_history(
-        self,
-        pair: str,
-        timeframe: str,
-        limit: int,
-        since_ms: Optional[int] = None,
-    ) -> List[List]:
-        """
-        Fetch funding rate history
-        Necessary workaround until https://github.com/ccxt/ccxt/issues/15990 is fixed.
-        """
-        params = {}
-        if since_ms:
-            until = since_ms + (timeframe_to_msecs(timeframe) * self._ft_has['ohlcv_candle_limit'])
-            params.update({'until': until})
-        # Funding rate
-        data = await self._api_async.fetch_funding_rate_history(
-            pair, since=since_ms,
-            params=params)
-        # Convert funding rate to candle pattern
-        data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data]
-        return data
+    def ohlcv_candle_limit(
+            self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
+
+        if candle_type in (CandleType.FUNDING_RATE):
+            return 200
+
+        return super().ohlcv_candle_limit(timeframe, candle_type, since_ms)
 
     def _lev_prep(self, pair: str, leverage: float, side: BuySell, accept_fail: bool = False):
         if self.trading_mode != TradingMode.SPOT:
@@ -218,3 +205,31 @@ class Bybit(Exchange):
             return self._fetch_and_calculate_funding_fees(
                     pair, amount, is_short, open_date)
         return 0.0
+
+    def fetch_orders(self, pair: str, since: datetime, params: Optional[Dict] = None) -> List[Dict]:
+        """
+        Fetch all orders for a pair "since"
+        :param pair: Pair for the query
+        :param since: Starting time for the query
+        """
+        # On bybit, the distance between since and "until" can't exceed 7 days.
+        # we therefore need to split the query into multiple queries.
+        orders = []
+
+        while since < dt_now():
+            until = since + timedelta(days=7, minutes=-1)
+            orders += super().fetch_orders(pair, since, params={'until': dt_ts(until)})
+            since = until
+
+        return orders
+
+    def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict:
+        order = super().fetch_order(order_id, pair, params)
+        if (
+            order.get('status') == 'canceled'
+            and order.get('filled') == 0.0
+            and order.get('remaining') == 0.0
+        ):
+            # Canceled orders will have "remaining=0" on bybit.
+            order['remaining'] = None
+        return order

freqtrade/exchange/exchange.py

@@ -5,6 +5,7 @@ Cryptocurrency Exchanges support
 import asyncio
 import inspect
 import logging
+import signal
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from math import floor
@@ -22,8 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN
                                  BuySell, Config, EntryExit, ExchangeConfig,
                                  ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe)
 from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
-from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode
-from freqtrade.enums.pricetype import PriceType
+from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
@@ -61,7 +61,8 @@ class Exchange:
     # or by specifying them in the configuration.
     _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
-        "stop_price_param": "stopPrice",
+        "stop_price_param": "stopLossPrice",  # Used for stoploss_on_exchange request
+        "stop_price_prop": "stopLossPrice",  # Used for stoploss_on_exchange response parsing
         "order_time_in_force": ["GTC"],
         "ohlcv_params": {},
         "ohlcv_candle_limit": 500,
@@ -263,8 +264,6 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(f"Initialization of ccxt failed. Reason: {e}") from e
 
-        self.set_sandbox(api, exchange_config, name)
-
         return api
 
     @property
@@ -465,16 +464,6 @@ class Exchange:
         return amount_to_contract_precision(amount, self.get_precision_amount(pair),
                                             self.precisionMode, contract_size)
 
-    def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None:
-        if exchange_config.get('sandbox'):
-            if api.urls.get('test'):
-                api.urls['api'] = api.urls['test']
-                logger.info("Enabled Sandbox API on %s", name)
-            else:
-                logger.warning(
-                    f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json")
-                raise OperationalException(f'Exchange {name} does not provide a sandbox api')
-
     def _load_async_markets(self, reload: bool = False) -> None:
         try:
             if self._api_async:
@@ -580,7 +569,7 @@ class Exchange:
         for pair in [f"{curr_1}/{curr_2}", f"{curr_2}/{curr_1}"]:
             if pair in self.markets and self.markets[pair].get('active'):
                 return pair
-        raise ExchangeError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.")
+        raise ValueError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.")
 
     def validate_timeframes(self, timeframe: Optional[str]) -> None:
         """
@@ -843,7 +832,7 @@ class Exchange:
                              rate: float, leverage: float, params: Dict = {},
                              stop_loss: bool = False) -> Dict[str, Any]:
         now = dt_now()
-        order_id = f'dry_run_{side}_{now.timestamp()}'
+        order_id = f'dry_run_{side}_{pair}_{now.timestamp()}'
         # Rounding here must respect to contract sizes
         _amount = self._contracts_to_amount(
             pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount)))
@@ -867,15 +856,15 @@ class Exchange:
         }
         if stop_loss:
             dry_order["info"] = {"stopPrice": dry_order["price"]}
-            dry_order[self._ft_has['stop_price_param']] = dry_order["price"]
+            dry_order[self._ft_has['stop_price_prop']] = dry_order["price"]
             # Workaround to avoid filling stoploss orders immediately
             dry_order["ft_order_type"] = "stoploss"
         orderbook: Optional[OrderBook] = None
         if self.exchange_has('fetchL2OrderBook'):
             orderbook = self.fetch_l2_order_book(pair, 20)
         if ordertype == "limit" and orderbook:
-            # Allow a 3% price difference
-            allowed_diff = 0.03
+            # Allow a 1% price difference
+            allowed_diff = 0.01
             if self._dry_is_price_crossed(pair, side, rate, orderbook, allowed_diff):
                 logger.info(
                     f"Converted order {pair} to market order due to price {rate} crossing spread "
@@ -931,7 +920,7 @@ class Exchange:
             max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
 
             remaining_amount = amount
-            filled_amount = 0.0
+            filled_value = 0.0
             book_entry_price = 0.0
             for book_entry in orderbook[ob_type]:
                 book_entry_price = book_entry[0]
@@ -939,17 +928,17 @@ class Exchange:
                 if remaining_amount > 0:
                     if remaining_amount < book_entry_coin_volume:
                         # Orderbook at this slot bigger than remaining amount
-                        filled_amount += remaining_amount * book_entry_price
+                        filled_value += remaining_amount * book_entry_price
                         break
                     else:
-                        filled_amount += book_entry_coin_volume * book_entry_price
+                        filled_value += book_entry_coin_volume * book_entry_price
                     remaining_amount -= book_entry_coin_volume
                 else:
                     break
             else:
                 # If remaining_amount wasn't consumed completely (break was not called)
-                filled_amount += remaining_amount * book_entry_price
-            forecast_avg_filled_price = max(filled_amount, 0) / amount
+                filled_value += remaining_amount * book_entry_price
+            forecast_avg_filled_price = max(filled_value, 0) / amount
             # Limit max. slippage to specified value
             if side == 'buy':
                 forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val)
@@ -1019,7 +1008,7 @@ class Exchange:
             from freqtrade.persistence import Order
             order = Order.order_by_id(order_id)
             if order:
-                ccxt_order = order.to_ccxt_object(self._ft_has['stop_price_param'])
+                ccxt_order = order.to_ccxt_object(self._ft_has['stop_price_prop'])
                 self._dry_run_open_orders[order_id] = ccxt_order
                 return ccxt_order
             # Gracefully handle errors with dry-run orders.
@@ -1091,6 +1080,13 @@ class Exchange:
                 rate_for_order,
                 params,
             )
+            if order.get('status') is None:
+                # Map empty status to open.
+                order['status'] = 'open'
+
+            if order.get('type') is None:
+                order['type'] = ordertype
+
             self._log_exchange_response('create_order', order)
             order = self._order_contracts_to_amount(order)
             return order
@@ -1120,7 +1116,7 @@ class Exchange:
         """
         if not self._ft_has.get('stoploss_on_exchange'):
             raise OperationalException(f"stoploss is not implemented for {self.name}.")
-        price_param = self._ft_has['stop_price_param']
+        price_param = self._ft_has['stop_price_prop']
         return (
             order.get(price_param, None) is None
             or ((side == "sell" and stop_loss > float(order[price_param])) or
@@ -1432,8 +1428,17 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]:
+        orders = []
+        if self.exchange_has('fetchClosedOrders'):
+            orders = self._api.fetch_closed_orders(pair, since=since_ms)
+            if self.exchange_has('fetchOpenOrders'):
+                orders_open = self._api.fetch_open_orders(pair, since=since_ms)
+                orders.extend(orders_open)
+        return orders
+
     @retrier(retries=0)
-    def fetch_orders(self, pair: str, since: datetime) -> List[Dict]:
+    def fetch_orders(self, pair: str, since: datetime, params: Optional[Dict] = None) -> List[Dict]:
         """
         Fetch all orders for a pair "since"
         :param pair: Pair for the query
@@ -1442,26 +1447,20 @@ class Exchange:
         if self._config['dry_run']:
             return []
 
-        def fetch_orders_emulate() -> List[Dict]:
-            orders = []
-            if self.exchange_has('fetchClosedOrders'):
-                orders = self._api.fetch_closed_orders(pair, since=since_ms)
-                if self.exchange_has('fetchOpenOrders'):
-                    orders_open = self._api.fetch_open_orders(pair, since=since_ms)
-                    orders.extend(orders_open)
-            return orders
-
         try:
             since_ms = int((since.timestamp() - 10) * 1000)
+
             if self.exchange_has('fetchOrders'):
+                if not params:
+                    params = {}
                 try:
-                    orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms)
+                    orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms, params=params)
                 except ccxt.NotSupported:
                     # Some exchanges don't support fetchOrders
                     # attempt to fetch open and closed orders separately
-                    orders = fetch_orders_emulate()
+                    orders = self._fetch_orders_emulate(pair, since_ms)
             else:
-                orders = fetch_orders_emulate()
+                orders = self._fetch_orders_emulate(pair, since_ms)
             self._log_exchange_response('fetch_orders', orders)
             orders = [self._order_contracts_to_amount(o) for o in orders]
             return orders
@@ -1876,7 +1875,7 @@ class Exchange:
                 tick = self.fetch_ticker(comb)
 
                 fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask')
-            except ExchangeError:
+            except (ValueError, ExchangeError):
                 fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate', None)
                 if not fee_to_quote_rate:
                     return None
@@ -2163,7 +2162,7 @@ class Exchange:
             except IndexError:
                 logger.exception("Error loading %s. Result was %s.", pair, data)
                 return pair, timeframe, candle_type, [], self._ohlcv_partial_candle
-            logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe)
+            logger.debug("Done fetching pair %s, %s interval %s...", pair, candle_type, timeframe)
             return pair, timeframe, candle_type, data, self._ohlcv_partial_candle
 
         except ccxt.NotSupported as e:
@@ -2265,20 +2264,24 @@ class Exchange:
             from_id = t[-1][1]
             trades.extend(t[:-1])
         while True:
-            t = await self._async_fetch_trades(pair,
-                                               params={self._trades_pagination_arg: from_id})
-            if t:
-                # Skip last id since its the key for the next call
-                trades.extend(t[:-1])
-                if from_id == t[-1][1] or t[-1][0] > until:
-                    logger.debug(f"Stopping because from_id did not change. "
-                                 f"Reached {t[-1][0]} > {until}")
-                    # Reached the end of the defined-download period - add last trade as well.
-                    trades.extend(t[-1:])
-                    break
+            try:
+                t = await self._async_fetch_trades(pair,
+                                                   params={self._trades_pagination_arg: from_id})
+                if t:
+                    # Skip last id since its the key for the next call
+                    trades.extend(t[:-1])
+                    if from_id == t[-1][1] or t[-1][0] > until:
+                        logger.debug(f"Stopping because from_id did not change. "
+                                     f"Reached {t[-1][0]} > {until}")
+                        # Reached the end of the defined-download period - add last trade as well.
+                        trades.extend(t[-1:])
+                        break
 
-                from_id = t[-1][1]
-            else:
+                    from_id = t[-1][1]
+                else:
+                    break
+            except asyncio.CancelledError:
+                logger.debug("Async operation Interrupted, breaking trades DL loop.")
                 break
 
         return (pair, trades)
@@ -2298,16 +2301,20 @@ class Exchange:
         # DEFAULT_TRADES_COLUMNS: 0 -> timestamp
         # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
-            t = await self._async_fetch_trades(pair, since=since)
-            if t:
-                since = t[-1][0]
-                trades.extend(t)
-                # Reached the end of the defined-download period
-                if until and t[-1][0] > until:
-                    logger.debug(
-                        f"Stopping because until was reached. {t[-1][0]} > {until}")
+            try:
+                t = await self._async_fetch_trades(pair, since=since)
+                if t:
+                    since = t[-1][0]
+                    trades.extend(t)
+                    # Reached the end of the defined-download period
+                    if until and t[-1][0] > until:
+                        logger.debug(
+                            f"Stopping because until was reached. {t[-1][0]} > {until}")
+                        break
+                else:
                     break
-            else:
+            except asyncio.CancelledError:
+                logger.debug("Async operation Interrupted, breaking trades DL loop.")
                 break
 
         return (pair, trades)
@@ -2356,9 +2363,16 @@ class Exchange:
             raise OperationalException("This exchange does not support downloading Trades.")
 
         with self._loop_lock:
-            return self.loop.run_until_complete(
-                self._async_get_trade_history(pair=pair, since=since,
-                                              until=until, from_id=from_id))
+            task = asyncio.ensure_future(self._async_get_trade_history(
+                pair=pair, since=since, until=until, from_id=from_id))
+
+            for sig in [signal.SIGINT, signal.SIGTERM]:
+                try:
+                    self.loop.add_signal_handler(sig, task.cancel)
+                except NotImplementedError:
+                    # Not all platforms implement signals (e.g. windows)
+                    pass
+            return self.loop.run_until_complete(task)
 
     @retrier
     def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float:

freqtrade/exchange/exchange_utils.py

@@ -248,6 +248,39 @@ def amount_to_contract_precision(
     return amount
 
 
+def __price_to_precision_significant_digits(
+    price: float,
+    price_precision: float,
+    *,
+    rounding_mode: int = ROUND,
+) -> float:
+    """
+    Implementation of ROUND_UP/Round_down for significant digits mode.
+    """
+    from decimal import ROUND_DOWN as dec_ROUND_DOWN
+    from decimal import ROUND_UP as dec_ROUND_UP
+    from decimal import Decimal
+    dec = Decimal(str(price))
+    string = f'{dec:f}'
+    precision = round(price_precision)
+
+    q = precision - dec.adjusted() - 1
+    sigfig = Decimal('10') ** -q
+    if q < 0:
+        string_to_precision = string[:precision]
+        # string_to_precision is '' when we have zero precision
+        below = sigfig * Decimal(string_to_precision if string_to_precision else '0')
+        above = below + sigfig
+        res = above if rounding_mode == ROUND_UP else below
+        precise = f'{res:f}'
+    else:
+        precise = '{:f}'.format(dec.quantize(
+            sigfig,
+            rounding=dec_ROUND_DOWN if rounding_mode == ROUND_DOWN else dec_ROUND_UP)
+        )
+    return float(precise)
+
+
 def price_to_precision(
     price: float,
     price_precision: Optional[float],
@@ -271,28 +304,39 @@ def price_to_precision(
     :return: price rounded up to the precision the Exchange accepts
     """
     if price_precision is not None and precisionMode is not None:
+        if rounding_mode not in (ROUND_UP, ROUND_DOWN):
+            # Use CCXT code where possible.
+            return float(decimal_to_precision(price, rounding_mode=rounding_mode,
+                                              precision=price_precision,
+                                              counting_mode=precisionMode
+                                              ))
+
         if precisionMode == TICK_SIZE:
-            if rounding_mode == ROUND:
-                ticks = price / price_precision
-                rounded_ticks = round(ticks)
-                return rounded_ticks * price_precision
             precision = FtPrecise(price_precision)
             price_str = FtPrecise(price)
             missing = price_str % precision
             if not missing == FtPrecise("0"):
-                return round(float(str(price_str - missing + precision)), 14)
+                if rounding_mode == ROUND_UP:
+                    res = price_str - missing + precision
+                elif rounding_mode == ROUND_DOWN:
+                    res = price_str - missing
+                return round(float(str(res)), 14)
             return price
-        elif precisionMode in (SIGNIFICANT_DIGITS, DECIMAL_PLACES):
+        elif precisionMode == DECIMAL_PLACES:
+
             ndigits = round(price_precision)
-            if rounding_mode == ROUND:
-                return round(price, ndigits)
             ticks = price * (10**ndigits)
             if rounding_mode == ROUND_UP:
                 return ceil(ticks) / (10**ndigits)
-            if rounding_mode == TRUNCATE:
-                return int(ticks) / (10**ndigits)
             if rounding_mode == ROUND_DOWN:
                 return floor(ticks) / (10**ndigits)
+
             raise ValueError(f"Unknown rounding_mode {rounding_mode}")
+        elif precisionMode == SIGNIFICANT_DIGITS:
+            if rounding_mode in (ROUND_UP, ROUND_DOWN):
+                return __price_to_precision_significant_digits(
+                    price, price_precision, rounding_mode=rounding_mode
+                )
+
         raise ValueError(f"Unknown precisionMode {precisionMode}")
     return price

freqtrade/exchange/gate.py

@@ -25,8 +25,10 @@ class Gate(Exchange):
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
         "order_time_in_force": ['GTC', 'IOC'],
-        "stoploss_order_types": {"limit": "limit"},
         "stoploss_on_exchange": True,
+        "stoploss_order_types": {"limit": "limit"},
+        "stop_price_param": "stopPrice",
+        "stop_price_prop": "stopPrice",
         "marketOrderRequiresPrice": True,
     }
 

freqtrade/exchange/huobi.py

@@ -17,6 +17,8 @@ class Huobi(Exchange):
 
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
+        "stop_price_param": "stopPrice",
+        "stop_price_prop": "stopPrice",
         "stoploss_order_types": {"limit": "stop-limit"},
         "ohlcv_candle_limit": 1000,
         "l2_limit_range": [5, 10, 20],

freqtrade/exchange/kraken.py

@@ -24,6 +24,8 @@ class Kraken(Exchange):
     _params: Dict = {"trading_agreement": "agree"}
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
+        "stop_price_param": "stopPrice",
+        "stop_price_prop": "stopPrice",
         "ohlcv_candle_limit": 720,
         "ohlcv_has_history": False,
         "trades_pagination": "id",

freqtrade/exchange/kucoin.py

@@ -21,6 +21,8 @@ class Kucoin(Exchange):
 
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
+        "stop_price_param": "stopPrice",
+        "stop_price_prop": "stopPrice",
         "stoploss_order_types": {"limit": "limit", "market": "market"},
         "l2_limit_range": [20, 100],
         "l2_limit_range_required": False,

freqtrade/exchange/okx.py

@@ -1,16 +1,17 @@
 import logging
+from datetime import timedelta
 from typing import Any, Dict, List, Optional, Tuple
 
 import ccxt
 
 from freqtrade.constants import BuySell
-from freqtrade.enums import CandleType, MarginMode, TradingMode
-from freqtrade.enums.pricetype import PriceType
+from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode
 from freqtrade.exceptions import (DDosProtection, OperationalException, RetryableOrderError,
                                   TemporaryError)
 from freqtrade.exchange import Exchange, date_minus_candles
 from freqtrade.exchange.common import retrier
 from freqtrade.misc import safe_value_fallback2
+from freqtrade.util import dt_now, dt_ts
 
 
 logger = logging.getLogger(__name__)
@@ -28,7 +29,6 @@ class Okx(Exchange):
         "funding_fee_timeframe": "8h",
         "stoploss_order_types": {"limit": "limit"},
         "stoploss_on_exchange": True,
-        "stop_price_param": "stopLossPrice",
     }
     _ft_has_futures: Dict = {
         "tickers_have_quoteVolume": False,
@@ -187,7 +187,7 @@ class Okx(Exchange):
 
     def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict:
         if (
-            order['status'] == 'closed'
+            order.get('status', 'open') == 'closed'
             and (real_order_id := order.get('info', {}).get('ordId')) is not None
         ):
             # Once a order triggered, we fetch the regular followup order.
@@ -241,3 +241,18 @@ class Okx(Exchange):
             pair=pair,
             params=params1,
         )
+
+    def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]:
+        orders = []
+
+        orders = self._api.fetch_closed_orders(pair, since=since_ms)
+        if (since_ms < dt_ts(dt_now() - timedelta(days=6, hours=23))):
+            # Regular fetch_closed_orders only returns 7 days of data.
+            # Force usage of "archive" endpoint, which returns 3 months of data.
+            params = {'method': 'privateGetTradeOrdersHistoryArchive'}
+            orders_hist = self._api.fetch_closed_orders(pair, since=since_ms, params=params)
+            orders.extend(orders_hist)
+
+        orders_open = self._api.fetch_open_orders(pair, since=since_ms)
+        orders.extend(orders_open)
+        return orders

freqtrade/freqai/RL/BaseEnvironment.py

@@ -11,6 +11,8 @@ from gymnasium import spaces
 from gymnasium.utils import seeding
 from pandas import DataFrame
 
+from freqtrade.exceptions import OperationalException
+
 
 logger = logging.getLogger(__name__)
 
@@ -80,8 +82,9 @@ class BaseEnvironment(gym.Env):
         self.can_short: bool = can_short
         self.live: bool = live
         if not self.live and self.add_state_info:
-            self.add_state_info = False
-            logger.warning("add_state_info is not available in backtesting. Deactivating.")
+            raise OperationalException("`add_state_info` is not available in backtesting. Change "
+                                       "parameter to false in your rl_config. See `add_state_info` "
+                                       "docs for more info.")
         self.seed(seed)
         self.reset_env(df, prices, window_size, reward_kwargs, starting_point)
 

freqtrade/freqai/RL/BaseReinforcementLearningModel.py

@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
 torch.multiprocessing.set_sharing_strategy('file_system')
 
 SB3_MODELS = ['PPO', 'A2C', 'DQN']
-SB3_CONTRIB_MODELS = ['TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO']
+SB3_CONTRIB_MODELS = ['TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'QRDQN']
 
 
 class BaseReinforcementLearningModel(IFreqaiModel):

freqtrade/freqai/data_drawer.py

@@ -263,23 +263,46 @@ class FreqaiDataDrawer:
             self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
             return
 
-    def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None:
+    def set_initial_return_values(self, pair: str,
+                                  pred_df: DataFrame,
+                                  dataframe: DataFrame
+                                  ) -> None:
         """
         Set the initial return values to the historical predictions dataframe. This avoids needing
         to repredict on historical candles, and also stores historical predictions despite
         retrainings (so stored predictions are true predictions, not just inferencing on trained
-        data)
+        data).
+
+        We also aim to keep the date from historical predictions so that the FreqUI displays
+        zeros during any downtime (between FreqAI reloads).
         """
 
-        hist_df = self.historic_predictions
-        len_diff = len(hist_df[pair].index) - len(pred_df.index)
-        if len_diff < 0:
-            df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
-                                  ignore_index=True, keys=hist_df[pair].keys())
+        new_pred = pred_df.copy()
+        # set new_pred values to nans (we want to signal to user that there was nothing
+        # historically made during downtime. The newest pred will get appeneded later in
+        # append_model_predictions)
+        new_pred.iloc[:, :] = np.nan
+        new_pred["date_pred"] = dataframe["date"]
+        hist_preds = self.historic_predictions[pair].copy()
+
+        # find the closest common date between new_pred and historic predictions
+        # and cut off the new_pred dataframe at that date
+        common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner")
+        if len(common_dates.index) > 0:
+            new_pred = new_pred.iloc[len(common_dates):]
         else:
-            df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
+            logger.warning("No common dates found between new predictions and historic "
+                           "predictions. You likely left your FreqAI instance offline "
+                           f"for more than {len(dataframe.index)} candles.")
+
+        df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys())
+        # remove last row because we will append that later in append_model_predictions()
+        df_concat = df_concat.iloc[:-1]
+        # any missing values will get zeroed out so users can see the exact
+        # downtime in FreqUI
         df_concat = df_concat.fillna(0)
-        self.model_return_values[pair] = df_concat
+        self.historic_predictions[pair] = df_concat
+        self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True)
 
     def append_model_predictions(self, pair: str, predictions: DataFrame,
                                  do_preds: NDArray[np.int_],
@@ -375,7 +398,7 @@ class FreqaiDataDrawer:
         num_keep = self.freqai_info["purge_old_models"]
         if not num_keep:
             return
-        elif type(num_keep) == bool:
+        elif isinstance(num_keep, bool):
             num_keep = 2
 
         model_folders = [x for x in self.full_path.iterdir() if x.is_dir()]
@@ -635,7 +658,7 @@ class FreqaiDataDrawer:
                     timeframe=tf,
                     pair=pair,
                     timerange=timerange,
-                    data_format=self.config.get("dataformat_ohlcv", "json"),
+                    data_format=self.config.get("dataformat_ohlcv", "feather"),
                     candle_type=self.config.get("candle_type_def", CandleType.SPOT),
                 )
 

freqtrade/freqai/data_kitchen.py

@@ -244,6 +244,14 @@ class FreqaiDataKitchen:
                 f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points"
                 f" due to NaNs in populated dataset {len(unfiltered_df)}."
             )
+            if len(unfiltered_df) == 0 and not self.live:
+                raise OperationalException(
+                    f"{self.pair}: all training data dropped due to NaNs. "
+                    "You likely did not download enough training data prior "
+                    "to your backtest timerange. Hint:\n"
+                    f"{DOCS_LINK}/freqai-running/"
+                    "#downloading-data-to-cover-the-full-backtest-period"
+                )
             if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live:
                 worst_indicator = str(unfiltered_df.count().idxmin())
                 logger.warning(

freqtrade/freqai/freqai_interface.py

@@ -138,7 +138,6 @@ class IFreqaiModel(ABC):
         :param metadata: pair metadata coming from strategy.
         :param strategy: Strategy to train on
         """
-
         self.live = strategy.dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE)
         self.dd.set_pair_dict_info(metadata)
         self.data_provider = strategy.dp
@@ -394,6 +393,11 @@ class IFreqaiModel(ABC):
         dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
         """
 
+        if not strategy.process_only_new_candles:
+            raise OperationalException("You are trying to use a FreqAI strategy with "
+                                       "process_only_new_candles = False. This is not supported "
+                                       "by FreqAI, and it is therefore aborting.")
+
         # get the model metadata associated with the current pair
         (_, trained_timestamp) = self.dd.get_pair_dict_info(metadata["pair"])
 
@@ -453,7 +457,7 @@ class IFreqaiModel(ABC):
             pred_df, do_preds = self.predict(dataframe, dk)
             if pair not in self.dd.historic_predictions:
                 self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
-            self.dd.set_initial_return_values(pair, pred_df)
+            self.dd.set_initial_return_values(pair, pred_df, dataframe)
 
             dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
             return
@@ -645,11 +649,11 @@ class IFreqaiModel(ABC):
         If the user reuses an identifier on a subsequent instance,
         this function will not be called. In that case, "real" predictions
         will be appended to the loaded set of historic predictions.
-        :param df: DataFrame = the dataframe containing the training feature data
-        :param model: Any = A model which was `fit` using a common library such as
-                      catboost or lightgbm
+        :param pred_df: DataFrame = the dataframe containing the predictions coming
+            out of a model
         :param dk: FreqaiDataKitchen = object containing methods for data analysis
         :param pair: str = current pair
+        :param strat_df: DataFrame = dataframe coming from strategy
         """
 
         self.dd.historic_predictions[pair] = pred_df

freqtrade/freqai/prediction_models/PyTorchMLPClassifier.py

@@ -26,9 +26,9 @@ class PyTorchMLPClassifier(BasePyTorchClassifier):
             "model_training_parameters" : {
                 "learning_rate": 3e-4,
                 "trainer_kwargs": {
-                    "max_iters": 5000,
+                    "n_steps": 5000,
                     "batch_size": 64,
-                    "max_n_eval_batches": null,
+                    "n_epochs": null,
                 },
                 "model_kwargs": {
                     "hidden_dim": 512,

freqtrade/freqai/prediction_models/PyTorchMLPRegressor.py

@@ -27,9 +27,9 @@ class PyTorchMLPRegressor(BasePyTorchRegressor):
             "model_training_parameters" : {
                 "learning_rate": 3e-4,
                 "trainer_kwargs": {
-                    "max_iters": 5000,
+                    "n_steps": 5000,
                     "batch_size": 64,
-                    "max_n_eval_batches": null,
+                    "n_epochs": null,
                 },
                 "model_kwargs": {
                     "hidden_dim": 512,

freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py

@@ -30,9 +30,9 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor):
             "model_training_parameters" : {
                 "learning_rate": 3e-4,
                 "trainer_kwargs": {
-                    "max_iters": 5000,
+                    "n_steps": 5000,
                     "batch_size": 64,
-                    "max_n_eval_batches": null
+                    "n_epochs": null
                 },
                 "model_kwargs": {
                     "hidden_dim": 512,

freqtrade/freqai/torch/PyTorchDataConvertor.py

@@ -1,5 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import Optional
 
 import pandas as pd
 import torch
@@ -12,14 +11,14 @@ class PyTorchDataConvertor(ABC):
     """
 
     @abstractmethod
-    def convert_x(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor:
+    def convert_x(self, df: pd.DataFrame, device: str) -> torch.Tensor:
         """
         :param df: "*_features" dataframe.
         :param device: The device to use for training (e.g. 'cpu', 'cuda').
         """
 
     @abstractmethod
-    def convert_y(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor:
+    def convert_y(self, df: pd.DataFrame, device: str) -> torch.Tensor:
         """
         :param df: "*_labels" dataframe.
         :param device: The device to use for training (e.g. 'cpu', 'cuda').
@@ -33,8 +32,8 @@ class DefaultPyTorchDataConvertor(PyTorchDataConvertor):
 
     def __init__(
             self,
-            target_tensor_type: Optional[torch.dtype] = None,
-            squeeze_target_tensor: bool = False
+            target_tensor_type: torch.dtype = torch.float32,
+            squeeze_target_tensor: bool = False,
     ):
         """
         :param target_tensor_type: type of target tensor, for classification use
@@ -45,23 +44,14 @@ class DefaultPyTorchDataConvertor(PyTorchDataConvertor):
         self._target_tensor_type = target_tensor_type
         self._squeeze_target_tensor = squeeze_target_tensor
 
-    def convert_x(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor:
-        x = torch.from_numpy(df.values).float()
-        if device:
-            x = x.to(device)
-
+    def convert_x(self, df: pd.DataFrame, device: str) -> torch.Tensor:
+        numpy_arrays = df.values
+        x = torch.tensor(numpy_arrays, device=device, dtype=torch.float32)
         return x
 
-    def convert_y(self, df: pd.DataFrame, device: Optional[str] = None) -> torch.Tensor:
-        y = torch.from_numpy(df.values)
-
-        if self._target_tensor_type:
-            y = y.to(self._target_tensor_type)
-
+    def convert_y(self, df: pd.DataFrame, device: str) -> torch.Tensor:
+        numpy_arrays = df.values
+        y = torch.tensor(numpy_arrays, device=device, dtype=self._target_tensor_type)
         if self._squeeze_target_tensor:
             y = y.squeeze()
-
-        if device:
-            y = y.to(device)
-
         return y

freqtrade/freqai/torch/PyTorchModelTrainer.py

@@ -1,5 +1,4 @@
 import logging
-import math
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -40,23 +39,27 @@ class PyTorchModelTrainer(PyTorchTrainerInterface):
             state_dict and model_meta_data saved by self.save() method.
         :param model_meta_data: Additional metadata about the model (optional).
         :param data_convertor: convertor from pd.DataFrame to torch.tensor.
-        :param max_iters: The number of training iterations to run.
-            iteration here refers to the number of times we call
-            self.optimizer.step(). used to calculate n_epochs.
+        :param n_steps: used to calculate n_epochs. The number of training iterations to run.
+            iteration here refers to the number of times optimizer.step() is called.
+            ignored if n_epochs is set.
+        :param n_epochs: The maximum number batches to use for evaluation.
         :param batch_size: The size of the batches to use during training.
-        :param max_n_eval_batches: The maximum number batches to use for evaluation.
         """
         self.model = model
         self.optimizer = optimizer
         self.criterion = criterion
         self.model_meta_data = model_meta_data
         self.device = device
-        self.max_iters: int = kwargs.get("max_iters", 100)
+        self.n_epochs: Optional[int] = kwargs.get("n_epochs", 10)
+        self.n_steps: Optional[int] = kwargs.get("n_steps", None)
+        if self.n_steps is None and not self.n_epochs:
+            raise Exception("Either `n_steps` or `n_epochs` should be set.")
+
         self.batch_size: int = kwargs.get("batch_size", 64)
-        self.max_n_eval_batches: Optional[int] = kwargs.get("max_n_eval_batches", None)
         self.data_convertor = data_convertor
         self.window_size: int = window_size
         self.tb_logger = tb_logger
+        self.test_batch_counter = 0
 
     def fit(self, data_dictionary: Dict[str, pd.DataFrame], splits: List[str]):
         """
@@ -72,55 +75,46 @@ class PyTorchModelTrainer(PyTorchTrainerInterface):
            backpropagation.
          - Updates the model's parameters using an optimizer.
         """
-        data_loaders_dictionary = self.create_data_loaders_dictionary(data_dictionary, splits)
-        epochs = self.calc_n_epochs(
-            n_obs=len(data_dictionary["train_features"]),
-            batch_size=self.batch_size,
-            n_iters=self.max_iters
-        )
         self.model.train()
-        for epoch in range(1, epochs + 1):
-            for i, batch_data in enumerate(data_loaders_dictionary["train"]):
 
+        data_loaders_dictionary = self.create_data_loaders_dictionary(data_dictionary, splits)
+        n_obs = len(data_dictionary["train_features"])
+        n_epochs = self.n_epochs or self.calc_n_epochs(n_obs=n_obs)
+        batch_counter = 0
+        for _ in range(n_epochs):
+            for _, batch_data in enumerate(data_loaders_dictionary["train"]):
                 xb, yb = batch_data
-                xb.to(self.device)
-                yb.to(self.device)
+                xb = xb.to(self.device)
+                yb = yb.to(self.device)
                 yb_pred = self.model(xb)
                 loss = self.criterion(yb_pred, yb)
 
                 self.optimizer.zero_grad(set_to_none=True)
                 loss.backward()
                 self.optimizer.step()
-                self.tb_logger.log_scalar("train_loss", loss.item(), i)
+                self.tb_logger.log_scalar("train_loss", loss.item(), batch_counter)
+                batch_counter += 1
 
             # evaluation
             if "test" in splits:
-                self.estimate_loss(
-                    data_loaders_dictionary,
-                    self.max_n_eval_batches,
-                    "test"
-                )
+                self.estimate_loss(data_loaders_dictionary, "test")
 
     @torch.no_grad()
     def estimate_loss(
             self,
             data_loader_dictionary: Dict[str, DataLoader],
-            max_n_eval_batches: Optional[int],
             split: str,
     ) -> None:
         self.model.eval()
-        n_batches = 0
-        for i, batch_data in enumerate(data_loader_dictionary[split]):
-            if max_n_eval_batches and i > max_n_eval_batches:
-                n_batches += 1
-                break
+        for _, batch_data in enumerate(data_loader_dictionary[split]):
             xb, yb = batch_data
-            xb.to(self.device)
-            yb.to(self.device)
+            xb = xb.to(self.device)
+            yb = yb.to(self.device)
 
             yb_pred = self.model(xb)
             loss = self.criterion(yb_pred, yb)
-            self.tb_logger.log_scalar(f"{split}_loss", loss.item(), i)
+            self.tb_logger.log_scalar(f"{split}_loss", loss.item(), self.test_batch_counter)
+            self.test_batch_counter += 1
 
         self.model.train()
 
@@ -148,31 +142,30 @@ class PyTorchModelTrainer(PyTorchTrainerInterface):
 
         return data_loader_dictionary
 
-    @staticmethod
-    def calc_n_epochs(n_obs: int, batch_size: int, n_iters: int) -> int:
+    def calc_n_epochs(self, n_obs: int) -> int:
         """
         Calculates the number of epochs required to reach the maximum number
         of iterations specified in the model training parameters.
 
-        the motivation here is that `max_iters` is easier to optimize and keep stable,
+        the motivation here is that `n_steps` is easier to optimize and keep stable,
         across different n_obs - the number of data points.
         """
+        assert isinstance(self.n_steps, int), "Either `n_steps` or `n_epochs` should be set."
+        n_batches = n_obs // self.batch_size
+        n_epochs = min(self.n_steps // n_batches, 1)
+        if n_epochs <= 10:
+            logger.warning(
+                f"Setting low n_epochs: {n_epochs}. "
+                f"Please consider increasing `n_steps` hyper-parameter."
+            )
 
-        n_batches = math.ceil(n_obs // batch_size)
-        epochs = math.ceil(n_iters // n_batches)
-        if epochs <= 10:
-            logger.warning("User set `max_iters` in such a way that the trainer will only perform "
-                           f" {epochs} epochs. Please consider increasing this value accordingly")
-            if epochs <= 1:
-                logger.warning("Epochs set to 1. Please review your `max_iters` value")
-                epochs = 1
-        return epochs
+        return n_epochs
 
     def save(self, path: Path):
         """
         - Saving any nn.Module state_dict
         - Saving model_meta_data, this dict should contain any additional data that the
-          user needs to store. e.g class_names for classification models.
+          user needs to store. e.g. class_names for classification models.
         """
 
         torch.save({

freqtrade/freqai/utils.py

@@ -50,7 +50,7 @@ def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
         timerange=timerange,
         new_pairs_days=new_pairs_days,
         erase=False,
-        data_format=config.get("dataformat_ohlcv", "json"),
+        data_format=config.get("dataformat_ohlcv", "feather"),
         trading_mode=config.get("trading_mode", "spot"),
         prepend=config.get("prepend_data", False),
     )