diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d187e650b..e18090f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ jobs: strategy: matrix: os: [ ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -127,10 +127,10 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -237,10 +237,10 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -304,7 +304,7 @@ jobs: mypy_version_check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -319,7 +319,7 @@ jobs: pre-commit: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: @@ -329,7 +329,7 @@ jobs: docs_check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Documentation syntax run: | @@ -359,7 +359,7 @@ jobs: # Run pytest with "live" checks runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 @@ -443,12 +443,12 @@ jobs: if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Extract branch name shell: bash @@ -515,7 +515,7 @@ jobs: if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Extract branch name shell: bash diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 4587626f6..9d2f8b768 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -8,7 +8,7 @@ jobs: dockerHubDescription: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Docker Hub Description uses: peter-evans/dockerhub-description@v3 env: diff --git a/.gitignore b/.gitignore index 758a324f4..400a082f3 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,9 @@ instance/ # Scrapy stuff: .scrapy +# memray +memray-* + # Sphinx documentation docs/_build/ # Mkdocs documentation diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bbf75965..5c7a1728b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,17 +8,17 @@ repos: # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.5.0" + rev: "v1.5.1" hooks: - id: mypy exclude: build_helpers additional_dependencies: - types-cachetools==5.3.0.6 - types-filelock==3.2.7 - - types-requests==2.31.0.2 + - types-requests==2.31.0.4 - types-tabulate==0.9.0.3 - types-python-dateutil==2.8.19.14 - - SQLAlchemy==2.0.20 + - SQLAlchemy==2.0.21 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/Dockerfile b/Dockerfile index b5f6f5d5e..7a4e85812 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.4-slim-bullseye as base +FROM python:3.11.5-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/README.md b/README.md index 57c4e3a52..0cacfe703 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Please find the complete documentation on the [freqtrade website](https://www.fr ## Features -- [x] **Based on Python 3.8+**: For botting on any operating system - Windows, macOS and Linux. +- [x] **Based on Python 3.9+**: For botting on any operating system - Windows, macOS and Linux. - [x] **Persistence**: Persistence is achieved through sqlite. - [x] **Dry-run**: Run the bot without paying money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. @@ -207,7 +207,7 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements -- [Python >= 3.8](http://docs.python-guide.org/en/latest/starting/installation/) +- [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - [TA-Lib](https://ta-lib.github.io/ta-lib-python/) diff --git a/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl deleted file mode 100644 index 34fecd677..000000000 Binary files a/build_helpers/TA_Lib-0.4.28-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/pyarrow-12.0.1-cp39-cp39-linux_armv7l.whl b/build_helpers/pyarrow-13.0.0-cp39-cp39-linux_armv7l.whl similarity index 63% rename from build_helpers/pyarrow-12.0.1-cp39-cp39-linux_armv7l.whl rename to build_helpers/pyarrow-13.0.0-cp39-cp39-linux_armv7l.whl index 55211ca01..6dcca7db8 100644 Binary files a/build_helpers/pyarrow-12.0.1-cp39-cp39-linux_armv7l.whl and b/build_helpers/pyarrow-13.0.0-cp39-cp39-linux_armv7l.whl differ diff --git a/config_examples/config_bittrex.example.json b/config_examples/config_bittrex.example.json index 3be5ba092..882b92edc 100644 --- a/config_examples/config_bittrex.example.json +++ b/config_examples/config_bittrex.example.json @@ -32,11 +32,8 @@ "name": "bittrex", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 500 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ "ETH/BTC", "LTC/BTC", diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 4681ec7df..cb2d4797e 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -70,6 +70,7 @@ }, "pairlists": [ {"method": "StaticPairList"}, + {"method": "FullTradesFilter"}, { "method": "VolumePairList", "number_assets": 20, diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 4972e7109..902cab9aa 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -36,8 +36,9 @@ ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies COPY --chown=ftuser:ftuser requirements.txt /freqtrade/ USER ftuser -RUN pip install --user --no-cache-dir numpy \ +RUN pip install --user --no-cache-dir numpy==1.25.2 \ && pip install --user /tmp/pyarrow-*.whl \ + && pip install --user --no-build-isolation TA-Lib==0.4.28 \ && pip install --user --no-cache-dir -r requirements.txt # Copy dependencies to runtime-image diff --git a/docs/configuration.md b/docs/configuration.md index 6c0795c67..7303f78c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -177,7 +177,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exit_pricing.order_book_top` | Bot will use the top N rate in Order Book "price_side" to exit. I.e. a value of 2 will allow the bot to pick the 2nd ask rate in [Order Book Exit](#exit-price-with-orderbook-enabled)
*Defaults to `1`.*
**Datatype:** Positive Integer | `custom_price_max_distance_ratio` | Configure maximum distance ratio between current and custom entry or exit price.
*Defaults to `0.02` 2%).*
**Datatype:** Positive float | | **TODO** -| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean +| `use_exit_signal` | Use exit signals produced by the strategy in addition to the `minimal_roi`.
Setting this to false disables the usage of `"exit_long"` and `"exit_short"` columns. Has no influence on other exit methods (Stoploss, ROI, callbacks). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `exit_profit_only` | Wait until the bot reaches `exit_profit_offset` before taking an exit decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `exit_profit_offset` | Exit-signal is only active above this value. Only active in combination with `exit_profit_only=True`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0`.*
**Datatype:** Float (as ratio) | `ignore_roi_if_entry_signal` | Do not exit if the entry signal is still active. This setting takes preference over `minimal_roi` and `use_exit_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean @@ -613,6 +613,7 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo * Orders are simulated, and will not be posted to the exchange. * Market orders fill based on orderbook volume the moment the order is placed. * Limit orders fill once the price reaches the defined level - or time out based on `unfilledtimeout` settings. +* Limit orders will be converted to market orders if they cross the price by more than 1%. * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. * Open orders (not trades, which are stored in the database) are kept open after bot restarts, with the assumption that they were not filled while being offline. diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 7f5637dd0..200aa289d 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -10,7 +10,7 @@ You can run this server using the following command: `docker compose -f docker/d This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. Please use the link that's printed in the console after startup for simplified login. -For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section. +For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analysis-using-docker-compose) section. ### Pro tips diff --git a/docs/data-download.md b/docs/data-download.md index 27ee4d377..890a89b60 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -154,13 +154,13 @@ freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT -- Freqtrade currently supports the following data-formats: +* `feather` - a dataformat based on Apache Arrow * `json` - plain "text" json files * `jsongz` - a gzip-zipped version of json files * `hdf5` - a high performance datastore -* `feather` - a dataformat based on Apache Arrow * `parquet` - columnar datastore (OHLCV only) -By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. +By default, both OHLCV data and trades data are stored in the `feather` format. This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: @@ -203,15 +203,15 @@ time freqtrade list-data --show-timerange --data-format-ohlcv | Format | Size | timing | |------------|-------------|-------------| +| `feather` | 72Mb | 3.5s | | `json` | 149Mb | 25.6s | | `jsongz` | 39Mb | 27s | | `hdf5` | 145Mb | 3.9s | -| `feather` | 72Mb | 3.5s | | `parquet` | 83Mb | 3.8s | Size has been taken from the BTC/USDT 1m spot combination for the timerange specified above. -To have a best performance/size mix, we recommend the use of either feather or parquet. +To have a best performance/size mix, we recommend using the default feather format, or parquet. ### Pairs file diff --git a/docs/edge.md b/docs/edge.md index e92abf40f..bb702f202 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -2,6 +2,10 @@ The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. +!!! Danger "Deprecated functionality" + `Edge positioning` (or short Edge) is currently in maintenance mode only (we keep existing functionality alive) and should be considered as deprecated. + It will currently not receive new features until either someone stepped forward to take up ownership of that module - or we'll decide to remove edge from freqtrade. + !!! Warning When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. diff --git a/docs/exchanges.md b/docs/exchanges.md index fb3049ba5..ab42cb9cf 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -55,7 +55,7 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t ## Binance !!! Warning "Server location and geo-ip restrictions" - Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. + Please be aware that Binance restricts API access regarding the server country. The current and non-exhaustive countries blocked are Canada, Malaysia, Netherlands and United States. Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. Binance supports [time_in_force](configuration.md#understand-order_time_in_force). @@ -136,15 +136,6 @@ Freqtrade will not attempt to change these settings. The Kraken API does only provide 720 historic candles, which is sufficient for Freqtrade dry-run and live trade modes, but is a problem for backtesting. To download data for the Kraken exchange, using `--dl-trades` is mandatory, otherwise the bot will download the same 720 candles over and over, and you'll not have enough backtest data. -Due to the heavy rate-limiting applied by Kraken, the following configuration section should be used to download data: - -``` json - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 3100 - }, -``` - !!! Warning "Downloading data from kraken" Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine. It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient. diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index a781d0834..b7a30eb5a 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -178,7 +178,7 @@ You can ask for each of the defined features to be included also for informative `include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells FreqAI to include the past 2 candles for each of the features in the feature set. -In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` +In total, the number of features the user of the presented example strategy has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` $= 3 * 3 * 3 * 2 * 2 = 108$. !!! note "Learn more about creative feature engineering" diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index df4508c86..c5cda3bc3 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -237,11 +237,10 @@ class MyCoolRLModel(ReinforcementLearner): Reinforcement Learning models benefit from tracking training metrics. FreqAI has integrated Tensorboard to allow users to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command: ```bash -cd freqtrade tensorboard --logdir user_data/models/unique-id ``` -where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in their browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard). +where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell to view the output in the browser at 127.0.0.1:6006 (6006 is the default port used by Tensorboard). ![tensorboard](assets/tensorboard.jpg) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9cdcc9bca..b3b23d6ff 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -25,6 +25,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`ProducerPairList`](#producerpairlist) * [`RemotePairList`](#remotepairlist) * [`AgeFilter`](#agefilter) +* [`FullTradesFilter`](#fulltradesfilter) * [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) @@ -236,6 +237,17 @@ be caught out buying before the pair has finished dropping in price. This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`. +#### FullTradesFilter + +Shrink whitelist to consist only in-trade pairs when the trade slots are full (when `max_open_trades` isn't being set to `-1` in the config). + +When the trade slots are full, there is no need to calculate indicators of the rest of the pairs (except informative pairs) since no new trade can be opened. By shrinking the whitelist to just the in-trade pairs, you can improve calculation speeds and reduce CPU usage. When a trade slot is free (either a trade is closed or `max_open_trades` value in config is increased), then the whitelist will return to normal state. + +When multiple pairlist filters are being used, it's recommended to put this filter at second position directly below the primary pairlist, so when the trade slots are full, the bot doesn't have to download data for the rest of the filters. + +!!! Warning "Backtesting" + `FullTradesFilter` does not support backtesting mode. + #### OffsetFilter Offsets an incoming pairlist by a given `offset` value. @@ -376,7 +388,7 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from "lookback_days": 10, "min_rate_of_change": 0.01, "max_rate_of_change": 0.99, - "refresh_period": 1440 + "refresh_period": 86400 } ] ``` @@ -431,7 +443,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01, - "refresh_period": 1440 + "refresh_period": 86400 }, { "method": "VolatilityFilter", diff --git a/docs/index.md b/docs/index.md index 77542ae78..190e7e3c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ To run this bot we recommend you a linux cloud instance with a minimum of: Alternatively -- Python 3.8+ +- Python 3.9+ - pip (pip3) - git - TA-Lib diff --git a/docs/installation.md b/docs/installation.md index eab0171c5..a87a3ff4e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -24,7 +24,7 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note - Python3.8 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Python3.9 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. Also, python headers (`python-dev` / `python-devel`) must be available for the installation to complete successfully. !!! Warning "Up-to-date clock" @@ -42,7 +42,7 @@ These requirements apply to both [Script Installation](#script-installation) and ### Install guide -* [Python >= 3.8.x](http://docs.python-guide.org/en/latest/starting/installation/) +* [Python >= 3.9](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation.html) (Recommended) @@ -54,7 +54,7 @@ We've included/collected install instructions for Ubuntu, MacOS, and Windows. Th OS Specific steps are listed first, the [Common](#common) section below is necessary for all systems. !!! Note - Python3.8 or higher and the corresponding pip are assumed to be available. + Python3.9 or higher and the corresponding pip are assumed to be available. === "Debian/Ubuntu" #### Install necessary dependencies @@ -169,7 +169,7 @@ You can as well update, configure and reset the codebase of your bot with `./scr ** --install ** With this option, the script will install the bot and most dependencies: -You will need to have git and python3.8+ installed beforehand for this to work. +You will need to have git and python3.9+ installed beforehand for this to work. * Mandatory software as: `ta-lib` * Setup your virtualenv under `.venv/` diff --git a/docs/recursive-analysis.md b/docs/recursive-analysis.md new file mode 100644 index 000000000..512b79f8f --- /dev/null +++ b/docs/recursive-analysis.md @@ -0,0 +1,89 @@ +# Recursive analysis + +This page explains how to validate your strategy for inaccuracies due to recursive issues with certain indicators. + +A recursive formula defines any term of a sequence relative to its preceding term(s). An example of a recursive formula is an = an-1 + b. + +Why does this matter for Freqtrade? In backtesting, the bot will get full data of the pairs according to the timerange specified. But in a dry/live run, the bot will be limited by the amount of data each exchanges gives. + +For example, to calculate a very basic indicator called `steps`, the first row's value is always 0, while the following rows' values are equal to the value of the previous row plus 1. If I were to calculate it using the latest 1000 candles, then the `steps` value of the first row is 0, and the `steps` value at the last closed candle is 999. + +What happens if the calculation is using only the latest 500 candles? Then instead of 999, the `steps` value at last closed candle is 499. The difference of the value means your backtest result can differ from your dry/live run result. + +The `recursive-analysis` command requires historic data to be available. To learn how to get data for the pairs and exchange you're interested in, +head over to the [Data Downloading](data-download.md) section of the documentation. + +This command is built upon preparing different lengths of data and calculates indicators based on them. +This does not backtest the strategy itself, but rather only calculates the indicators. After calculating the indicators of different startup candle values (`startup_candle_count`) are done, the values of last rows across all specified `startup_candle_count` are compared to see how much variance they show compared to the base calculation. + +Command settings: + +- Use the `-p` option to set your desired pair to analyze. Since we are only looking at indicator values, using more than one pair is redundant. Preferably use a pair with a relatively high price and at least moderate volatility, such as BTC or ETH, to avoid rounding issues that can make the results inaccurate. If no pair is set on the command, the pair used for this analysis is the first pair in the whitelist. +- It is recommended to set a long timerange (at least 5000 candles) so that the initial indicators' calculation that is going to be used as a benchmark has very small or no recursive issues itself. For example, for a 5m timeframe, a timerange of 5000 candles would be equal to 18 days. +- `--cache` is forced to "none" to avoid loading previous indicators calculation automatically. + +In addition to the recursive formula check, this command also carries out a simple lookahead bias check on the indicator values only. For a full lookahead check, use [Lookahead-analysis](lookahead-analysis.md). + +## Recursive-analysis command reference + +``` +usage: freqtrade recursive-analysis [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] + [--recursive-strategy-search] + [--freqaimodel NAME] + [--freqaimodel-path PATH] [-i TIMEFRAME] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}] + [-p PAIR] + [--freqai-backtest-live-models] + [--startup-candle STARTUP_CANDLES [STARTUP_CANDLES ...]] + +optional arguments: +-p PAIR, --pairs PAIR + Limit command to this pair. +--startup-candle STARTUP_CANDLE [STARTUP_CANDLE ...] + Provide a space-separated list of startup_candle_count to + be checked. Default : `199 399 499 999 1999`. +``` + +### Why are odd-numbered default startup candles used? + +The default value for startup candles are odd numbers. When the bot fetches candle data from the exchange's API, the last candle is the one being checked by the bot and the rest of the data are the "startup candles". + +For example, Binance allows 1000 candles per API call. When the bot receives 1000 candles, the last candle is the "current candle", and the preceding 999 candles are the "startup candles". By setting the startup candle count as 1000 instead of 999, the bot will try to fetch 1001 candles instead. The exchange API will then send candle data in a paginated form, i.e. in case of the Binance API, this will be two groups- one of length 1000 and another of length 1. This results in the bot thinking the strategy needs 1001 candles of data, and so it will download 2000 candles worth of data instead, which means there will be 1 "current candle" and 1999 "startup candles". + +Furthermore, exchanges limit the number of consecutive bulk API calls, e.g. Binance allows 5 calls. In this case, only 5000 candles can be downloaded from Binance API without hitting the API rate limit, which means the max `startup_candle_count` you can have is 4999. + +Please note that this candle limit may be changed in the future by the exchanges without any prior notice. + +### How does the command work? + +- Firstly an initial indicator calculation is carried out using the supplied timerange to generate a benchmark for indicator values. +- After setting the benchmark it will then carry out additional runs for each of the different startup candle count values. +- The command will then compare the indicator values at the last candle rows and report the differences in a table. + +## Understanding the recursive-analysis output + +This is an example of an output results table where at least one indicator has a recursive formula issue: + +``` +| indicators | 20 | 40 | 80 | 100 | 150 | 300 | 999 | +|--------------+---------+---------+--------+--------+---------+---------+--------| +| rsi_30 | nan% | -6.025% | 0.612% | 0.828% | -0.140% | 0.000% | 0.000% | +| rsi_14 | 24.141% | -0.876% | 0.070% | 0.007% | -0.000% | -0.000% | - | +``` + +The column headers indicate the different `startup_candle_count` used in the analysis. The values in the table indicate the variance of the calculated indicators compared to the benchmark value. + +`nan%` means the value of that indicator cannot be calculated due to lack of data. In this example, you cannot calculate RSI with length 30 with just 21 candles (1 current candle + 20 startup candles). + +Users should assess the table per indicator to decide if the specified `startup_candle_count` results in a sufficiently small variance so that the indicator does not have any effect on entries and/or exits. + +As such, aiming for absolute zero variance (shown by `-` value) might not be the best option, because some indicators might require you to use such a long `startup_candle_count` to have zero variance. + +## Caveats + +- `recursive-analysis` will only calculate and compare the indicator values at the last row. The output table reports the percentage differences between the different startup candle count calculations and the original benchmark calculation. Whether it has any actual impact on your entries and exits is not included. +- The ideal scenario is that indicators will have no variance (or at least very close to 0%) despite the startup candle being varied. In reality, indicators such as EMA are using a recursive formula to calculate indicator values, so the goal is not necessarily to have zero percentage variance, but to have the variance low enough (and therefore `startup_candle_count` high enough) that the recursion inherent in the indicator will not have any real impact on trading decisions. +- `recursive-analysis` will only run calculations on `populate_indicators` and `@informative` decorator(s). If you put any indicator calculation on `populate_entry_trend` or `populate_exit_trend`, it won't be calculated. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 96e3dbf1d..545e11f91 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.4 -mkdocs==1.5.2 -mkdocs-material==9.2.1 +mkdocs==1.5.3 +mkdocs-material==9.4.1 mdx_truly_sane_lists==1.3 -pymdown-extensions==10.1 +pymdown-extensions==10.3 jinja2==3.1.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index 5b33bfa6f..666056a65 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -151,6 +151,8 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `performance` | Show performance of each finished trade grouped by pair. | `balance` | Show account balance per currency. | `daily ` | Shows profit or loss per day, over the last n days (n defaults to 7). +| `weekly ` | Shows profit or loss per week, over the last n days (n defaults to 4). +| `monthly ` | Shows profit or loss per month, over the last n days (n defaults to 3). | `stats` | Display a summary of profit / loss reasons as well as average holding times. | `whitelist` | Show the current whitelist. | `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index ab8eb9f98..2639d5521 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -164,6 +164,31 @@ E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoplo During backtesting, `current_rate` (and `current_profit`) are provided against the candle's high (or low for short trades) - while the resulting stoploss is evaluated against the candle's low (or high for short trades). The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price. +Returning None will be interpreted as "no desire to change", and is the only safe way to return when you'd like to not modify the stoploss. + +Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)). + +!!! Note "Use of dates" + All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support. + +!!! Tip "Trailing stoploss" + It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior. + +### Adjust stoploss after position adjustments + +Depending on your strategy, you may encounter the need to adjust the stoploss in both directions after a [position adjustment](#adjust-trade-position). +For this, freqtrade will make an additional call with `after_fill=True` after an order fills, which will allow the strategy to move the stoploss in any direction (also widening the gap between stoploss and current price, which is otherwise forbidden). + +!!! Note "backwards compatibility" + This call will only be made if the `after_fill` parameter is part of the function definition of your `custom_stoploss` function. + As such, this will not impact (and with that, surprise) existing, running strategies. + +### Custom stoploss examples + +The next section will show some examples on what's possible with the custom stoploss function. +Of course, many more things are possible, and all examples can be combined at will. + +#### Trailing stop via custom stoploss To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum reached price) you would use the following very simple method: @@ -179,7 +204,8 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -187,7 +213,7 @@ class AwesomeStrategy(IStrategy): For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ - When not implemented by a strategy, returns the initial stoploss value + When not implemented by a strategy, returns the initial stoploss value. Only called when use_custom_stoploss is set to True. :param pair: Pair that's currently analyzed @@ -195,25 +221,13 @@ class AwesomeStrategy(IStrategy): :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param after_fill: True if the stoploss is called after the order was filled. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the current rate + :return float: New stoploss value, relative to the current_rate """ return -0.04 ``` -Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)). - -!!! Note "Use of dates" - All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support. - -!!! Tip "Trailing stoploss" - It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior. - -### Custom stoploss examples - -The next section will show some examples on what's possible with the custom stoploss function. -Of course, many more things are possible, and all examples can be combined at will. - #### Time based trailing stop Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. @@ -229,14 +243,45 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. if current_time - timedelta(minutes=120) > trade.open_date_utc: return -0.05 elif current_time - timedelta(minutes=60) > trade.open_date_utc: return -0.10 - return 1 + return None +``` + +#### Time based trailing stop with after-fill adjustments + +Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. +If an additional order fills, set stoploss to -10% below the new `open_rate` ([Averaged across all entries](#position-adjust-calculations)). + +``` python +from datetime import datetime, timedelta +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: + + if after_fill: + # After an additional order, start with a stoploss of 10% below the new open rate + return stoploss_from_open(0.10, current_profit, is_short=trade.is_short, leverage=trade.leverage) + # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. + if current_time - timedelta(minutes=120) > trade.open_date_utc: + return -0.05 + elif current_time - timedelta(minutes=60) > trade.open_date_utc: + return -0.10 + return None ``` #### Different stoploss per pair @@ -255,7 +300,8 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: if pair in ('ETH/BTC', 'XRP/BTC'): return -0.10 @@ -281,7 +327,8 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: if current_profit < 0.04: return -1 # return a value bigger than the initial stoploss to keep using the initial stoploss @@ -314,7 +361,8 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: # evaluate highest to lowest, so that highest possible stop is used if current_profit > 0.40: @@ -325,7 +373,7 @@ class AwesomeStrategy(IStrategy): return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage) # return maximum stoploss value, keeping current stoploss price unchanged - return 1 + return None ``` #### Custom stoploss using an indicator from dataframe example @@ -342,7 +390,8 @@ class AwesomeStrategy(IStrategy): use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) last_candle = dataframe.iloc[-1].squeeze() @@ -355,7 +404,7 @@ class AwesomeStrategy(IStrategy): return stoploss_from_absolute(stoploss_price, current_rate, is_short=trade.is_short) # return maximum stoploss value, keeping current stoploss price unchanged - return 1 + return None ``` See [Dataframe access](strategy-advanced.md#dataframe-access) for more information about dataframe use in strategy callbacks. @@ -364,15 +413,89 @@ See [Dataframe access](strategy-advanced.md#dataframe-access) for more informati #### Stoploss relative to open price -Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss relative to the *open* price, we need to use `current_profit` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price. +Stoploss values returned from `custom_stoploss()` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the _entry_ price instead. +`stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point. -The helper function [`stoploss_from_open()`](strategy-customization.md#stoploss_from_open) can be used to convert from an open price relative stop, to a current price relative stop which can be returned from `custom_stoploss()`. +??? Example "Returning a stoploss relative to the open price from the custom stoploss function" + + Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`). + + If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. + + This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%). + + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, stoploss_from_open + + class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: + + # once the profit has risen above 10%, keep the stoploss at 7% above the open price + if current_profit > 0.10: + return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage) + + return 1 + + ``` + + Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. + +!!! Note + Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings. + This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade + is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `exit_reason` in + `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when + `current_profit < open_relative_stop`. #### Stoploss percentage from absolute price Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss at specified absolute price level, we need to use `stop_rate` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price. -The helper function [`stoploss_from_absolute()`](strategy-customization.md#stoploss_from_absolute) can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`. +The helper function `stoploss_from_absolute()` can be used to convert from an absolute price, to a current price relative stop which can be returned from `custom_stoploss()`. + +??? Example "Returning a stoploss using absolute price from the custom stoploss function" + + If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage)`. + For futures, we need to adjust the direction (up or down), as well as adjust for leverage, since the [`custom_stoploss`](strategy-callbacks.md#custom-stoploss) callback returns the ["risk for this trade"](stoploss.md#stoploss-and-leverage) - not the relative price movement. + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, stoploss_from_absolute, timeframe_to_prev_date + + class AwesomeStrategy(IStrategy): + + use_custom_stoploss = True + + def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['atr'] = ta.ATR(dataframe, timeperiod=14) + return dataframe + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) + candle = dataframe.iloc[-1].squeeze() + sign = 1 if trade.is_short else -1 + return stoploss_from_absolute(current_rate + (side * candle['atr'] * 2), + current_rate, is_short=trade.is_short, + leverage=trade.leverage) + + ``` + --- @@ -387,6 +510,9 @@ Each of these methods are called right before placing an order on the exchange. !!! Note If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration. +!!! Note + Using custom_entry_price, the Trade object will be available as soon as the first entry order associated with the trade is created, for the first entry, `trade` parameter value will be `None`. + ### Custom order entry and exit price example ``` python @@ -397,7 +523,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional['Trade'], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, @@ -700,7 +826,7 @@ class DigDeeperStrategy(IStrategy): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -709,8 +835,9 @@ class DigDeeperStrategy(IStrategy): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. @@ -793,6 +920,8 @@ Returning any other price will cancel the existing order, and replace it with a The trade open-date (`trade.open_date_utc`) will remain at the time of the very first order placed. Please make sure to be aware of this - and eventually adjust your logic in other callbacks to account for this, and use the date of the first filled order instead. +If the cancellation of the original order fails, then the order will not be replaced - though the order will most likely have been canceled on exchange. Having this happen on initial entries will result in the deletion of the order, while on position adjustment orders, it'll result in the trade size remaining as is. + !!! Warning "Regular timeout" Entry `unfilledtimeout` mechanism (as well as `check_entry_timeout()`) takes precedence over this. Entry Orders that are cancelled via the above methods will not have this callback called. Be sure to update timeout values to match your expectations. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 8913d787b..e23c3cc41 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -168,10 +168,12 @@ Most indicators have an instable startup period, in which they are either not av To account for this, the strategy can be assigned the `startup_candle_count` attribute. This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. In the case where a user includes higher timeframes with informative pairs, the `startup_candle_count` does not necessarily change. The value is the maximum period (in candles) that any of the informatives timeframes need to compute stable indicators. -In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. +You can use [recursive-analysis](recursive-analysis.md) to check and find the correct `startup_candle_count` to be used. + +In this example strategy, this should be set to 400 (`startup_candle_count = 400`), since the minimum needed history for ema100 calculation to make sure the value is correct is 400 candles. ``` python - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=400) ``` By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. @@ -193,11 +195,11 @@ Let's try to backtest 1 month (January 2019) of 5m candles using an example stra freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m ``` -Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00. +Assuming `startup_candle_count` is set to 400, backtesting knows it needs 400 candles to generate valid buy signals. It will load data from `20190101 - (400 * 5m)` - which is ~2018-12-30 11:40:00. If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note - If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-02 09:20:00. ### Entry signal rules @@ -264,7 +266,7 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram ### Exit signal rules Edit the method `populate_exit_trend()` into your strategy file to update your exit strategy. -The exit-signal is only used for exits if `use_exit_signal` is set to true in the configuration. +The exit-signal can be suppressed by setting `use_exit_signal` to false in the configuration or strategy. `use_exit_signal` will not influence [signal collision rules](#colliding-signals) - which will still apply and can prevent entries. It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. @@ -586,6 +588,67 @@ for more information. will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique! +### *merge_informative_pair()* + +This method helps you merge an informative pair to a regular dataframe without lookahead bias. +It's there to help you merge the dataframe in a safe and consistent way. + +Options: + +- Rename the columns for you to create unique columns +- Merge the dataframe without lookahead bias +- Forward-fill (optional) + +For a full sample, please refer to the [complete data provider example](#complete-data-provider-sample) below. + +All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion: + +!!! Example "Column renaming" + Assuming `inf_tf = '1d'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d' # from the informative dataframe + ``` + +??? Example "Column renaming - 1h" + Assuming `inf_tf = '1h'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe + ``` + +??? Example "Custom implementation" + A custom implementation for this is possible, and can be done as follows: + + ``` python + + # Shift date by 1 candle + # This is necessary since the data is always the "open date" + # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00 + minutes = timeframe_to_minutes(inf_tf) + # Only do this if the timeframes are different: + informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm') + + # Rename columns to be unique + informative.columns = [f"{col}_{inf_tf}" for col in informative.columns] + # Assuming inf_tf = '1d' - then the columns will now be: + # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d + + # Combine the 2 dataframes + # all indicators on the informative sample MUST be calculated before this point + dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left') + # FFill to have the 1d value available in every row throughout the day. + # Without this, comparisons would only work once per day. + dataframe = dataframe.ffill() + + ``` + +!!! Warning "Informative timeframe < timeframe" + Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide. + To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need). + ## Additional data (DataProvider) The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. @@ -810,146 +873,6 @@ class SampleStrategy(IStrategy): *** -## Helper functions - -### *merge_informative_pair()* - -This method helps you merge an informative pair to a regular dataframe without lookahead bias. -It's there to help you merge the dataframe in a safe and consistent way. - -Options: - -- Rename the columns for you to create unique columns -- Merge the dataframe without lookahead bias -- Forward-fill (optional) - -For a full sample, please refer to the [complete data provider example](#complete-data-provider-sample) below. - -All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion: - -!!! Example "Column renaming" - Assuming `inf_tf = '1d'` the resulting columns will be: - - ``` python - 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe - 'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d' # from the informative dataframe - ``` - -??? Example "Column renaming - 1h" - Assuming `inf_tf = '1h'` the resulting columns will be: - - ``` python - 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe - 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe - ``` - -??? Example "Custom implementation" - A custom implementation for this is possible, and can be done as follows: - - ``` python - - # Shift date by 1 candle - # This is necessary since the data is always the "open date" - # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00 - minutes = timeframe_to_minutes(inf_tf) - # Only do this if the timeframes are different: - informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm') - - # Rename columns to be unique - informative.columns = [f"{col}_{inf_tf}" for col in informative.columns] - # Assuming inf_tf = '1d' - then the columns will now be: - # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d - - # Combine the 2 dataframes - # all indicators on the informative sample MUST be calculated before this point - dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left') - # FFill to have the 1d value available in every row throughout the day. - # Without this, comparisons would only work once per day. - dataframe = dataframe.ffill() - - ``` - -!!! Warning "Informative timeframe < timeframe" - Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide. - To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need). - -*** - -### *stoploss_from_open()* - -Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the entry point instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point. - -??? Example "Returning a stoploss relative to the open price from the custom stoploss function" - - Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`). - - If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. - - This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%). - - - ``` python - - from datetime import datetime - from freqtrade.persistence import Trade - from freqtrade.strategy import IStrategy, stoploss_from_open - - class AwesomeStrategy(IStrategy): - - # ... populate_* methods - - use_custom_stoploss = True - - def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: - - # once the profit has risen above 10%, keep the stoploss at 7% above the open price - if current_profit > 0.10: - return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage) - - return 1 - - ``` - - Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. - -!!! Note - Providing invalid input to `stoploss_from_open()` may produce "CustomStoploss function did not return valid stoploss" warnings. - This may happen if `current_profit` parameter is below specified `open_relative_stop`. Such situations may arise when closing trade - is blocked by `confirm_trade_exit()` method. Warnings can be solved by never blocking stop loss sells by checking `exit_reason` in - `confirm_trade_exit()`, or by using `return stoploss_from_open(...) or 1` idiom, which will request to not change stop loss when - `current_profit < open_relative_stop`. - -### *stoploss_from_absolute()* - -In some situations it may be confusing to deal with stops relative to current rate. Instead, you may define a stoploss level using an absolute price. - -??? Example "Returning a stoploss using absolute price from the custom stoploss function" - - If we want to trail a stop price at 2xATR below current price we can call `stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short)`. - - ``` python - - from datetime import datetime - from freqtrade.persistence import Trade - from freqtrade.strategy import IStrategy, stoploss_from_absolute - - class AwesomeStrategy(IStrategy): - - use_custom_stoploss = True - - def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['atr'] = ta.ATR(dataframe, timeperiod=14) - return dataframe - - def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: - dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) - candle = dataframe.iloc[-1].squeeze() - return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short) - - ``` - ## Additional data (Wallets) The strategy provides access to the `wallets` object. This contains the current balances on the exchange. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 06dd33bc2..846c53238 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -167,7 +167,7 @@ trades.groupby("pair")["exit_reason"].value_counts() # Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day) from freqtrade.configuration import Configuration -from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats +from freqtrade.data.btanalysis import load_backtest_stats import plotly.express as px import pandas as pd @@ -178,20 +178,8 @@ import pandas as pd stats = load_backtest_stats(backtest_dir) strategy_stats = stats['strategy'][strategy] -dates = [] -profits = [] -for date_profit in strategy_stats['daily_profit']: - dates.append(date_profit[0]) - profits.append(date_profit[1]) - -equity = 0 -equity_daily = [] -for daily_profit in profits: - equity_daily.append(equity) - equity += float(daily_profit) - - -df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily}) +df = pd.DataFrame(columns=['dates','equity'], data=strategy_stats['daily_profit']) +df['equity_daily'] = df['equity'].cumsum() fig = px.line(df, x="dates", y="equity_daily") fig.show() diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index d00349d1d..9e6f56e49 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -280,7 +280,7 @@ After: ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return proposed_rate ``` @@ -311,12 +311,13 @@ After: ``` python hl_lines="5 7" def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, - current_rate: float, current_profit: float, **kwargs) -> float: + current_rate: float, current_profit: float, after_fill: bool, + **kwargs) -> Optional[float]: # once the profit has risen above 10%, keep the stoploss at 7% above the open price if current_profit > 0.10: return stoploss_from_open(0.07, current_profit, is_short=trade.is_short) - return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short) + return stoploss_from_absolute(current_rate - (candle['atr'] * 2), current_rate, is_short=trade.is_short, leverage=trade.leverage) ``` diff --git a/docs/windows_installation.md b/docs/windows_installation.md index db785a1fc..5ac3d5e3d 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -24,7 +24,7 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.8, 3.9, 3.10 and 3.11) and for 64bit Windows. +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows. These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade. Other versions must be downloaded from the above link. diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 66620fa2b..f396cd0dd 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.8' +__version__ = '2023.9' if 'dev' in __version__: from pathlib import Path diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index fc45bdf61..ed950fa01 100755 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -3,7 +3,7 @@ __main__.py for Freqtrade To launch Freqtrade as a module -> python -m freqtrade (with Python >= 3.8) +> python -m freqtrade (with Python >= 3.9) """ from freqtrade import main diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index b9346fd5f..98e7cb084 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -20,7 +20,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_f start_list_timeframes, start_show_trades) from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show, start_edge, start_hyperopt, - start_lookahead_analysis) + start_lookahead_analysis, + start_recursive_analysis) from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit from freqtrade.commands.strategy_utils_commands import start_strategy_update diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 5473e95e1..736ae49fc 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -122,6 +122,8 @@ ARGS_LOOKAHEAD_ANALYSIS = [ a for a in ARGS_BACKTEST if a not in ("position_stacking", "use_max_market_positions", 'cache') ] + ["minimum_trade_amount", "targeted_trade_amount", "lookahead_analysis_exportfilename"] +ARGS_RECURSIVE_ANALYSIS = ["timeframe", "timerange", "dataformat_ohlcv", "pairs", "startup_candle"] + class Arguments: """ @@ -206,8 +208,9 @@ class Arguments: start_list_strategies, start_list_timeframes, start_lookahead_analysis, start_new_config, start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_strategy_update, - start_test_pairlist, start_trading, start_webserver) + start_recursive_analysis, start_show_trades, + start_strategy_update, start_test_pairlist, start_trading, + start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -467,3 +470,14 @@ class Arguments: self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, parser=lookahead_analayis_cmd) + + # Add recursive_analysis subcommand + recursive_analayis_cmd = subparsers.add_parser( + 'recursive-analysis', + help="Check for potential recursive formula issue.", + parents=[_common_parser, _strategy_parser]) + + recursive_analayis_cmd.set_defaults(func=start_recursive_analysis) + + self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS, + parser=recursive_analayis_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 586318e30..dcbd9714a 100755 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -705,4 +705,9 @@ AVAILABLE_CLI_OPTIONS = { help="Use this csv-filename to store lookahead-analysis-results", type=str ), + "startup_candle": Arg( + '--startup-candle', + help='Specify startup candles to be checked (`199`, `499`, `999`, `1999`).', + nargs='+', + ), } diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index fdfef6316..bccc6ea9a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -5,12 +5,12 @@ from typing import Any, Dict from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config -from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format -from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main +from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, + convert_trades_to_ohlcv) +from freqtrade.data.history import download_data_main from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes -from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util.binance_mig import migrate_binance_futures_data @@ -53,28 +53,19 @@ def start_convert_trades(args: Dict[str, Any]) -> None: # Remove stake-currency to skip checks which are not relevant for datadownload config['stake_currency'] = '' - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") if 'timeframes' not in config: config['timeframes'] = DL_DATA_TIMEFRAMES # Init exchange exchange = ExchangeResolver.load_exchange(config, validate=False) # Manual validations of relevant settings - if not config['exchange'].get('skip_pair_validation', False): - exchange.validate_pairs(config['pairs']) - expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) - - logger.info(f"About to Convert pairs: {expanded_pairs}, " - f"intervals: {config['timeframes']} to {config['datadir']}") for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) + # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=expanded_pairs, timeframes=config['timeframes'], + pairs=config.get('pairs', []), timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index cdddf0fe5..0a63753d9 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -144,3 +144,15 @@ def start_lookahead_analysis(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) LookaheadAnalysisSubFunctions.start(config) + + +def start_recursive_analysis(args: Dict[str, Any]) -> None: + """ + Start the backtest recursive tester script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + RecursiveAnalysisSubFunctions.start(config) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 43ede568c..e5e4d28a0 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -490,6 +490,9 @@ class Configuration: self._args_to_config(config, argname='lookahead_analysis_exportfilename', logstring='Path to store lookahead-analysis-results: {}') + self._args_to_config(config, argname='startup_candle', + logstring='Startup candle to be used on recursive analysis: {}') + def _process_runmode(self, config: Config) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 733fa11b3..c864833c3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,7 +33,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', - 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', + 'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', @@ -77,7 +77,8 @@ DL_DATA_TIMEFRAMES = ['1m', '5m'] ENV_VAR_PREFIX = 'FREQTRADE__' -NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') +CANCELED_EXCHANGE_STATES = ('cancelled', 'canceled', 'expired') +NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ('closed',) # Define decimals per coin for outputs # Only used for outputs. @@ -177,6 +178,11 @@ CONF_SCHEMA = { 'minimum_trade_amount': {'type': 'number', 'default': 10}, 'targeted_trade_amount': {'type': 'number', 'default': 20}, 'lookahead_analysis_exportfilename': {'type': 'string'}, + 'startup_candle': { + 'type': 'array', + 'uniqueItems': True, + 'default': [199, 399, 499, 999, 1999], + }, 'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99}, 'backtest_breakdown': { 'type': 'array', @@ -688,6 +694,7 @@ CANCEL_REASON = { "CANCELLED_ON_EXCHANGE": "cancelled on exchange", "FORCE_EXIT": "forcesold", "REPLACE": "cancelled to be replaced by new limit order", + "REPLACE_FAILED": "failed to replace order, deleting Trade", "USER_CANCEL": "user requested order cancel" } @@ -709,3 +716,6 @@ Config = Dict[str, Any] # Exchange part of the configuration. ExchangeConfig = Dict[str, Any] IntOrInf = float + + +EntryExecuteMode = Literal['initial', 'pos_adjust', 'replace'] diff --git a/freqtrade/data/converter/__init__.py b/freqtrade/data/converter/__init__.py new file mode 100644 index 000000000..3918e49da --- /dev/null +++ b/freqtrade/data/converter/__init__.py @@ -0,0 +1,28 @@ +from freqtrade.data.converter.converter import (clean_ohlcv_dataframe, convert_ohlcv_format, + ohlcv_fill_up_missing_data, ohlcv_to_dataframe, + order_book_to_dataframe, reduce_dataframe_footprint, + trim_dataframe, trim_dataframes) +from freqtrade.data.converter.trade_converter import (convert_trades_format, + convert_trades_to_ohlcv, trades_convert_types, + trades_df_remove_duplicates, + trades_dict_to_list, trades_list_to_df, + trades_to_ohlcv) + + +__all__ = [ + 'clean_ohlcv_dataframe', + 'convert_ohlcv_format', + 'ohlcv_fill_up_missing_data', + 'ohlcv_to_dataframe', + 'order_book_to_dataframe', + 'reduce_dataframe_footprint', + 'trim_dataframe', + 'trim_dataframes', + 'convert_trades_format', + 'convert_trades_to_ohlcv', + 'trades_convert_types', + 'trades_df_remove_duplicates', + 'trades_dict_to_list', + 'trades_list_to_df', + 'trades_to_ohlcv', +] diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter/converter.py similarity index 75% rename from freqtrade/data/converter.py rename to freqtrade/data/converter/converter.py index 324ce5def..8d1401e88 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter/converter.py @@ -2,14 +2,13 @@ Functions to convert data from one format to another """ import logging -from typing import Dict, List +from typing import Dict import numpy as np import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, - Config, TradeList) +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, Config from freqtrade.enums import CandleType, TradingMode @@ -105,7 +104,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) df = dataframe.resample(resample_interval, on='date').agg(ohlcv_dict) # Forwardfill close for missing columns - df['close'] = df['close'].fillna(method='ffill') + df['close'] = df['close'].ffill() # Use close for "open, high, low" df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna( value={'open': df['close'], @@ -194,97 +193,6 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_df_remove_duplicates(trades: pd.DataFrame) -> pd.DataFrame: - """ - Removes duplicates from the trades DataFrame. - Uses pandas.DataFrame.drop_duplicates to remove duplicates based on the 'timestamp' column. - :param trades: DataFrame with the columns constants.DEFAULT_TRADES_COLUMNS - :return: DataFrame with duplicates removed based on the 'timestamp' column - """ - return trades.drop_duplicates(subset=['timestamp', 'id']) - - -def trades_dict_to_list(trades: List[Dict]) -> TradeList: - """ - Convert fetch_trades result into a List (to be more memory efficient). - :param trades: List of trades, as returned by ccxt.fetch_trades. - :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns - """ - return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] - - -def trades_convert_types(trades: DataFrame) -> DataFrame: - """ - Convert Trades dtypes and add 'date' column - """ - trades = trades.astype(TRADES_DTYPES) - trades['date'] = to_datetime(trades['timestamp'], unit='ms', utc=True) - return trades - - -def trades_list_to_df(trades: TradeList, convert: bool = True): - """ - convert trades list to dataframe - :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns - """ - if not trades: - df = DataFrame(columns=DEFAULT_TRADES_COLUMNS) - else: - df = DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) - - if convert: - df = trades_convert_types(df) - - return df - - -def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame: - """ - Converts trades list to OHLCV list - :param trades: List of trades, as returned by ccxt.fetch_trades. - :param timeframe: Timeframe to resample data to - :return: OHLCV Dataframe. - :raises: ValueError if no trades are provided - """ - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) - if trades.empty: - raise ValueError('Trade-list empty.') - df = trades.set_index('date', drop=True) - - df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() - df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() - df_new['date'] = df_new.index - # Drop 0 volume rows - df_new = df_new.dropna() - return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS] - - -def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): - """ - Convert trades from one format to another format. - :param config: Config dictionary - :param convert_from: Source format - :param convert_to: Target format - :param erase: Erase source data (does not apply if source and target format are identical) - """ - from freqtrade.data.history.idatahandler import get_datahandler - src = get_datahandler(config['datadir'], convert_from) - trg = get_datahandler(config['datadir'], convert_to) - - if 'pairs' not in config: - config['pairs'] = src.trades_get_pairs(config['datadir']) - logger.info(f"Converting trades for {config['pairs']}") - - for pair in config['pairs']: - data = src.trades_load(pair=pair) - logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) - if erase and convert_from != convert_to: - logger.info(f"Deleting source Trade data for {pair}.") - src.trades_purge(pair=pair) - - def convert_ohlcv_format( config: Config, convert_from: str, diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py new file mode 100644 index 000000000..398ddc85e --- /dev/null +++ b/freqtrade/data/converter/trade_converter.py @@ -0,0 +1,144 @@ +""" +Functions to convert data from one format to another +""" +import logging +from pathlib import Path +from typing import Dict, List + +import pandas as pd +from pandas import DataFrame, to_datetime + +from freqtrade.configuration import TimeRange +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, + Config, TradeList) +from freqtrade.enums import CandleType + + +logger = logging.getLogger(__name__) + + +def trades_df_remove_duplicates(trades: pd.DataFrame) -> pd.DataFrame: + """ + Removes duplicates from the trades DataFrame. + Uses pandas.DataFrame.drop_duplicates to remove duplicates based on the 'timestamp' column. + :param trades: DataFrame with the columns constants.DEFAULT_TRADES_COLUMNS + :return: DataFrame with duplicates removed based on the 'timestamp' column + """ + return trades.drop_duplicates(subset=['timestamp', 'id']) + + +def trades_dict_to_list(trades: List[Dict]) -> TradeList: + """ + Convert fetch_trades result into a List (to be more memory efficient). + :param trades: List of trades, as returned by ccxt.fetch_trades. + :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns + """ + return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] + + +def trades_convert_types(trades: DataFrame) -> DataFrame: + """ + Convert Trades dtypes and add 'date' column + """ + trades = trades.astype(TRADES_DTYPES) + trades['date'] = to_datetime(trades['timestamp'], unit='ms', utc=True) + return trades + + +def trades_list_to_df(trades: TradeList, convert: bool = True): + """ + convert trades list to dataframe + :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns + """ + if not trades: + df = DataFrame(columns=DEFAULT_TRADES_COLUMNS) + else: + df = DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) + + if convert: + df = trades_convert_types(df) + + return df + + +def trades_to_ohlcv(trades: DataFrame, timeframe: str) -> DataFrame: + """ + Converts trades list to OHLCV list + :param trades: List of trades, as returned by ccxt.fetch_trades. + :param timeframe: Timeframe to resample data to + :return: OHLCV Dataframe. + :raises: ValueError if no trades are provided + """ + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + if trades.empty: + raise ValueError('Trade-list empty.') + df = trades.set_index('date', drop=True) + + df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() + df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() + df_new['date'] = df_new.index + # Drop 0 volume rows + df_new = df_new.dropna() + return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS] + + +def convert_trades_to_ohlcv( + pairs: List[str], + timeframes: List[str], + datadir: Path, + timerange: TimeRange, + erase: bool = False, + data_format_ohlcv: str = 'feather', + data_format_trades: str = 'feather', + candle_type: CandleType = CandleType.SPOT +) -> None: + """ + Convert stored trades data to ohlcv data + """ + from freqtrade.data.history.idatahandler import get_datahandler + data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) + data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) + if not pairs: + pairs = data_handler_trades.trades_get_pairs(datadir) + + logger.info(f"About to convert pairs: '{', '.join(pairs)}', " + f"intervals: '{', '.join(timeframes)}' to {datadir}") + + for pair in pairs: + trades = data_handler_trades.trades_load(pair) + for timeframe in timeframes: + if erase: + if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') + try: + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) + except ValueError: + logger.exception(f'Could not convert {pair} to OHLCV.') + + +def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): + """ + Convert trades from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase source data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + + if 'pairs' not in config: + config['pairs'] = src.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + data = src.trades_load(pair=pair) + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(pair, data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source Trade data for {pair}.") + src.trades_purge(pair=pair) diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py index db3a7d3a4..ac5893585 100644 --- a/freqtrade/data/entryexitanalysis.py +++ b/freqtrade/data/entryexitanalysis.py @@ -119,8 +119,15 @@ def _do_group_table_output(bigdf, glist, csv_path: Path, to_csv=False, ): new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0) new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0) - new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss', - 'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss'] + new['exp_ratio'] = ( + ( + (1 + (new['avg_win'] / abs(new['avg_loss']))) * (new['wl_ratio_pct'] / 100) + ) - 1).fillna(0) + + new.columns = ['total_num_buys', 'wins', 'losses', + 'profit_abs_wins', 'profit_abs_loss', + 'profit_tot', 'wl_ratio_pct', + 'avg_win', 'avg_loss', 'exp_ratio'] sortcols = ['total_num_buys'] @@ -204,6 +211,7 @@ def prepare_results(analysed_trades, stratname, timerange=None): res_df = pd.DataFrame() for pair, trades in analysed_trades[stratname].items(): + trades.dropna(subset=['close_date'], inplace=True) res_df = pd.concat([res_df, trades], ignore_index=True) res_df = _select_rows_within_dates(res_df, timerange) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 5175195c5..1ad1060a4 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,9 +9,9 @@ from pandas import DataFrame, concat from freqtrade.configuration import TimeRange from freqtrade.constants import (DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS, DL_DATA_TIMEFRAMES, Config) -from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, - trades_df_remove_duplicates, trades_list_to_df, - trades_to_ohlcv) +from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_ohlcv, + ohlcv_to_dataframe, trades_df_remove_duplicates, + trades_list_to_df) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException @@ -429,36 +429,6 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: return pairs_not_available -def convert_trades_to_ohlcv( - pairs: List[str], - timeframes: List[str], - datadir: Path, - timerange: TimeRange, - erase: bool = False, - data_format_ohlcv: str = 'feather', - data_format_trades: str = 'feather', - candle_type: CandleType = CandleType.SPOT -) -> None: - """ - Convert stored trades data to ohlcv data - """ - data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) - data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) - - for pair in pairs: - trades = data_handler_trades.trades_load(pair) - for timeframe in timeframes: - if erase: - if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): - logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - try: - ohlcv = trades_to_ohlcv(trades, timeframe) - # Store ohlcv - data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) - except ValueError: - logger.exception(f'Could not convert {pair} to OHLCV.') - - def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]: """ Get the maximum common timerange for the given backtest data. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index e8be804cb..bd7ef9b20 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -21,6 +21,8 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, + "stop_price_param": "stopPrice", + "stop_price_prop": "stopPrice", "stoploss_order_types": {"limit": "stop_loss_limit"}, "order_time_in_force": ["GTC", "FOK", "IOC", "PO"], "ohlcv_candle_limit": 1000, diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index f191cdd31..80aa06e0c 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -2092,10 +2092,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.02, - "maxLeverage": 10.0, + "maxLeverage": 20.0, "info": { "bracket": "1", - "initialLeverage": "10", + "initialLeverage": "20", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.02", @@ -2108,10 +2108,10 @@ "minNotional": 5000.0, "maxNotional": 25000.0, "maintenanceMarginRate": 0.025, - "maxLeverage": 8.0, + "maxLeverage": 15.0, "info": { "bracket": "2", - "initialLeverage": "8", + "initialLeverage": "15", "notionalCap": "25000", "notionalFloor": "5000", "maintMarginRatio": "0.025", @@ -2124,10 +2124,10 @@ "minNotional": 25000.0, "maxNotional": 200000.0, "maintenanceMarginRate": 0.05, - "maxLeverage": 6.0, + "maxLeverage": 10.0, "info": { "bracket": "3", - "initialLeverage": "6", + "initialLeverage": "10", "notionalCap": "200000", "notionalFloor": "25000", "maintMarginRatio": "0.05", @@ -2186,13 +2186,13 @@ "tier": 7.0, "currency": "USDT", "minNotional": 3000000.0, - "maxNotional": 3500000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "7", "initialLeverage": "1", - "notionalCap": "3500000", + "notionalCap": "5000000", "notionalFloor": "3000000", "maintMarginRatio": "0.5", "cum": "898150.0" @@ -3339,6 +3339,104 @@ } } ], + "ARK/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 10000.0, + "maintenanceMarginRate": 0.03, + "maxLeverage": 10.0, + "info": { + "bracket": "1", + "initialLeverage": "10", + "notionalCap": "10000", + "notionalFloor": "0", + "maintMarginRatio": "0.03", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 10000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 8.0, + "info": { + "bracket": "2", + "initialLeverage": "8", + "notionalCap": "200000", + "notionalFloor": "10000", + "maintMarginRatio": "0.05", + "cum": "200.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10200.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "4", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "22700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "147700.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 3500000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "3500000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "897700.0" + } + } + ], "ARKM/USDT:USDT": [ { "tier": 1.0, @@ -12152,96 +12250,80 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 8.0, "info": { "bracket": "1", - "initialLeverage": "20", - "notionalCap": "5000", + "initialLeverage": "8", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.025", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 6.0, "info": { "bracket": "2", - "initialLeverage": "10", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" + "initialLeverage": "6", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" } }, { "tier": 3.0, "currency": "USDT", - "minNotional": 25000.0, - "maxNotional": 100000.0, - "maintenanceMarginRate": 0.05, - "maxLeverage": 8.0, - "info": { - "bracket": "3", - "initialLeverage": "8", - "notionalCap": "100000", - "notionalFloor": "25000", - "maintMarginRatio": "0.05", - "cum": "650.0" - } - }, - { - "tier": 4.0, - "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5650.0" + "cum": "5625.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11900.0" + "cum": "11875.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 3000000.0, + "maxNotional": 1500000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "1", - "notionalCap": "3000000", + "notionalCap": "1500000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386900.0" + "cum": "386875.0" } } ], @@ -12441,6 +12523,104 @@ } } ], + "FRONT/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 10000.0, + "maintenanceMarginRate": 0.03, + "maxLeverage": 10.0, + "info": { + "bracket": "1", + "initialLeverage": "10", + "notionalCap": "10000", + "notionalFloor": "0", + "maintMarginRatio": "0.03", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 10000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 8.0, + "info": { + "bracket": "2", + "initialLeverage": "8", + "notionalCap": "200000", + "notionalFloor": "10000", + "maintMarginRatio": "0.05", + "cum": "200.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10200.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "4", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "22700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "147700.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 3500000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "3500000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "897700.0" + } + } + ], "FTM/BUSD:BUSD": [ { "tier": 1.0, @@ -13387,6 +13567,104 @@ } } ], + "GLMR/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 10000.0, + "maintenanceMarginRate": 0.03, + "maxLeverage": 10.0, + "info": { + "bracket": "1", + "initialLeverage": "10", + "notionalCap": "10000", + "notionalFloor": "0", + "maintMarginRatio": "0.03", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 10000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 8.0, + "info": { + "bracket": "2", + "initialLeverage": "8", + "notionalCap": "200000", + "notionalFloor": "10000", + "maintMarginRatio": "0.05", + "cum": "200.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10200.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "4", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "22700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "147700.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 3500000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "3500000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "897700.0" + } + } + ], "GMT/BUSD:BUSD": [ { "tier": 1.0, @@ -13946,13 +14224,13 @@ "tier": 2.0, "currency": "USDT", "minNotional": 5000.0, - "maxNotional": 25000.0, + "maxNotional": 50000.0, "maintenanceMarginRate": 0.025, "maxLeverage": 20.0, "info": { "bracket": "2", "initialLeverage": "20", - "notionalCap": "25000", + "notionalCap": "50000", "notionalFloor": "5000", "maintMarginRatio": "0.025", "cum": "75.0" @@ -13961,65 +14239,81 @@ { "tier": 3.0, "currency": "USDT", - "minNotional": 25000.0, - "maxNotional": 100000.0, + "minNotional": 50000.0, + "maxNotional": 200000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "3", "initialLeverage": "10", - "notionalCap": "100000", - "notionalFloor": "25000", + "notionalCap": "200000", + "notionalFloor": "50000", "maintMarginRatio": "0.05", - "cum": "700.0" + "cum": "1325.0" } }, { "tier": 4.0, "currency": "USDT", - "minNotional": 100000.0, - "maxNotional": 250000.0, + "minNotional": 200000.0, + "maxNotional": 500000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "4", "initialLeverage": "5", - "notionalCap": "250000", - "notionalFloor": "100000", + "notionalCap": "500000", + "notionalFloor": "200000", "maintMarginRatio": "0.1", - "cum": "5700.0" + "cum": "11325.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 250000.0, + "minNotional": 500000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, - "maxLeverage": 2.0, + "maxLeverage": 4.0, "info": { "bracket": "5", - "initialLeverage": "2", + "initialLeverage": "4", "notionalCap": "1000000", - "notionalFloor": "250000", + "notionalFloor": "500000", "maintMarginRatio": "0.125", - "cum": "11950.0" + "cum": "23825.0" } }, { "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "6", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148825.0" + } + }, + { + "tier": 7.0, + "currency": "USDT", + "minNotional": 3000000.0, "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "7", "initialLeverage": "1", "notionalCap": "5000000", - "notionalFloor": "1000000", + "notionalFloor": "3000000", "maintMarginRatio": "0.5", - "cum": "386950.0" + "cum": "898825.0" } } ], @@ -14137,6 +14431,104 @@ } } ], + "HIFI/USDT:USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 15.0, + "info": { + "bracket": "1", + "initialLeverage": "15", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 200000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "200000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 200000.0, + "maxNotional": 500000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "500000", + "notionalFloor": "200000", + "maintMarginRatio": "0.1", + "cum": "10625.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 500000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 4.0, + "info": { + "bracket": "4", + "initialLeverage": "4", + "notionalCap": "1000000", + "notionalFloor": "500000", + "maintMarginRatio": "0.125", + "cum": "23125.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 3000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "3000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.25", + "cum": "148125.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 3000000.0, + "maxNotional": 3500000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "3500000", + "notionalFloor": "3000000", + "maintMarginRatio": "0.5", + "cum": "898125.0" + } + } + ], "HIGH/USDT:USDT": [ { "tier": 1.0, @@ -15006,112 +15398,96 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.01, - "maxLeverage": 25.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, "info": { "bracket": "1", - "initialLeverage": "25", - "notionalCap": "5000", + "initialLeverage": "20", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.01", + "maintMarginRatio": "0.025", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 20.0, - "info": { - "bracket": "2", - "initialLeverage": "20", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "75.0" - } - }, - { - "tier": 3.0, - "currency": "USDT", "minNotional": 25000.0, "maxNotional": 600000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { - "bracket": "3", + "bracket": "2", "initialLeverage": "10", "notionalCap": "600000", "notionalFloor": "25000", "maintMarginRatio": "0.05", - "cum": "700.0" + "cum": "625.0" } }, { - "tier": 4.0, + "tier": 3.0, "currency": "USDT", "minNotional": 600000.0, "maxNotional": 1600000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "1600000", "notionalFloor": "600000", "maintMarginRatio": "0.1", - "cum": "30700.0" + "cum": "30625.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 1600000.0, "maxNotional": 2000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "4", "notionalCap": "2000000", "notionalFloor": "1600000", "maintMarginRatio": "0.125", - "cum": "70700.0" + "cum": "70625.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 2000000.0, + "maxNotional": 4000000.0, + "maintenanceMarginRate": 0.25, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "4000000", + "notionalFloor": "2000000", + "maintMarginRatio": "0.25", + "cum": "320625.0" } }, { "tier": 6.0, "currency": "USDT", - "minNotional": 2000000.0, - "maxNotional": 6000000.0, - "maintenanceMarginRate": 0.25, - "maxLeverage": 2.0, - "info": { - "bracket": "6", - "initialLeverage": "2", - "notionalCap": "6000000", - "notionalFloor": "2000000", - "maintMarginRatio": "0.25", - "cum": "320700.0" - } - }, - { - "tier": 7.0, - "currency": "USDT", - "minNotional": 6000000.0, - "maxNotional": 10000000.0, + "minNotional": 4000000.0, + "maxNotional": 4500000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "7", + "bracket": "6", "initialLeverage": "1", - "notionalCap": "10000000", - "notionalFloor": "6000000", + "notionalCap": "4500000", + "notionalFloor": "4000000", "maintMarginRatio": "0.5", - "cum": "1820700.0" + "cum": "1320625.0" } } ], @@ -15136,13 +15512,13 @@ "tier": 2.0, "currency": "USDT", "minNotional": 5000.0, - "maxNotional": 25000.0, + "maxNotional": 50000.0, "maintenanceMarginRate": 0.025, "maxLeverage": 20.0, "info": { "bracket": "2", "initialLeverage": "20", - "notionalCap": "25000", + "notionalCap": "50000", "notionalFloor": "5000", "maintMarginRatio": "0.025", "cum": "25.0" @@ -15151,39 +15527,39 @@ { "tier": 3.0, "currency": "USDT", - "minNotional": 25000.0, - "maxNotional": 1200000.0, + "minNotional": 50000.0, + "maxNotional": 1600000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "3", "initialLeverage": "10", - "notionalCap": "1200000", - "notionalFloor": "25000", + "notionalCap": "1600000", + "notionalFloor": "50000", "maintMarginRatio": "0.05", - "cum": "650.0" + "cum": "1275.0" } }, { "tier": 4.0, "currency": "USDT", - "minNotional": 1200000.0, - "maxNotional": 3200000.0, + "minNotional": 1600000.0, + "maxNotional": 3600000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "4", "initialLeverage": "5", - "notionalCap": "3200000", - "notionalFloor": "1200000", + "notionalCap": "3600000", + "notionalFloor": "1600000", "maintMarginRatio": "0.1", - "cum": "60650.0" + "cum": "81275.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 3200000.0, + "minNotional": 3600000.0, "maxNotional": 4000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, @@ -15191,9 +15567,9 @@ "bracket": "5", "initialLeverage": "4", "notionalCap": "4000000", - "notionalFloor": "3200000", + "notionalFloor": "3600000", "maintMarginRatio": "0.125", - "cum": "140650.0" + "cum": "171275.0" } }, { @@ -15209,7 +15585,7 @@ "notionalCap": "12000000", "notionalFloor": "4000000", "maintMarginRatio": "0.25", - "cum": "640650.0" + "cum": "671275.0" } }, { @@ -15225,7 +15601,7 @@ "notionalCap": "20000000", "notionalFloor": "12000000", "maintMarginRatio": "0.5", - "cum": "3640650.0" + "cum": "3671275.0" } } ], @@ -20642,112 +21018,96 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 16.0, "info": { "bracket": "1", - "initialLeverage": "20", - "notionalCap": "5000", + "initialLeverage": "16", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.025", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 15.0, - "info": { - "bracket": "2", - "initialLeverage": "15", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" - } - }, - { - "tier": 3.0, - "currency": "USDT", "minNotional": 25000.0, "maxNotional": 200000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { - "bracket": "3", + "bracket": "2", "initialLeverage": "10", "notionalCap": "200000", "notionalFloor": "25000", "maintMarginRatio": "0.05", - "cum": "650.0" + "cum": "625.0" } }, { - "tier": 4.0, + "tier": 3.0, "currency": "USDT", "minNotional": 200000.0, "maxNotional": 500000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "500000", "notionalFloor": "200000", "maintMarginRatio": "0.1", - "cum": "10650.0" + "cum": "10625.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 500000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "4", "notionalCap": "1000000", "notionalFloor": "500000", "maintMarginRatio": "0.125", - "cum": "23150.0" + "cum": "23125.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, "maxNotional": 3000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "2", "notionalCap": "3000000", "notionalFloor": "1000000", "maintMarginRatio": "0.25", - "cum": "148150.0" + "cum": "148125.0" } }, { - "tier": 7.0, + "tier": 6.0, "currency": "USDT", "minNotional": 3000000.0, - "maxNotional": 5000000.0, + "maxNotional": 3005000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "7", + "bracket": "6", "initialLeverage": "1", - "notionalCap": "5000000", + "notionalCap": "3005000", "notionalFloor": "3000000", "maintMarginRatio": "0.5", - "cum": "898150.0" + "cum": "898125.0" } } ], @@ -22712,13 +23072,13 @@ "tier": 3.0, "currency": "USDT", "minNotional": 25000.0, - "maxNotional": 200000.0, + "maxNotional": 300000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 10.0, "info": { "bracket": "3", "initialLeverage": "10", - "notionalCap": "200000", + "notionalCap": "300000", "notionalFloor": "25000", "maintMarginRatio": "0.05", "cum": "650.0" @@ -22727,39 +23087,39 @@ { "tier": 4.0, "currency": "USDT", - "minNotional": 200000.0, - "maxNotional": 500000.0, + "minNotional": 300000.0, + "maxNotional": 700000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { "bracket": "4", "initialLeverage": "5", - "notionalCap": "500000", - "notionalFloor": "200000", + "notionalCap": "700000", + "notionalFloor": "300000", "maintMarginRatio": "0.1", - "cum": "10650.0" + "cum": "15650.0" } }, { "tier": 5.0, "currency": "USDT", - "minNotional": 500000.0, - "maxNotional": 1000000.0, + "minNotional": 700000.0, + "maxNotional": 1200000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 4.0, "info": { "bracket": "5", "initialLeverage": "4", - "notionalCap": "1000000", - "notionalFloor": "500000", + "notionalCap": "1200000", + "notionalFloor": "700000", "maintMarginRatio": "0.125", - "cum": "23150.0" + "cum": "33150.0" } }, { "tier": 6.0, "currency": "USDT", - "minNotional": 1000000.0, + "minNotional": 1200000.0, "maxNotional": 3000000.0, "maintenanceMarginRate": 0.25, "maxLeverage": 2.0, @@ -22767,9 +23127,9 @@ "bracket": "6", "initialLeverage": "2", "notionalCap": "3000000", - "notionalFloor": "1000000", + "notionalFloor": "1200000", "maintMarginRatio": "0.25", - "cum": "148150.0" + "cum": "183150.0" } }, { @@ -22785,7 +23145,7 @@ "notionalCap": "5000000", "notionalFloor": "3000000", "maintMarginRatio": "0.5", - "cum": "898150.0" + "cum": "933150.0" } } ], @@ -23773,14 +24133,14 @@ "currency": "USDT", "minNotional": 0.0, "maxNotional": 5000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 8.0, + "maintenanceMarginRate": 0.02, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "8", + "initialLeverage": "10", "notionalCap": "5000", "notionalFloor": "0", - "maintMarginRatio": "0.025", + "maintMarginRatio": "0.02", "cum": "0.0" } }, @@ -23788,64 +24148,80 @@ "tier": 2.0, "currency": "USDT", "minNotional": 5000.0, - "maxNotional": 100000.0, - "maintenanceMarginRate": 0.05, - "maxLeverage": 6.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 8.0, "info": { "bracket": "2", - "initialLeverage": "6", - "notionalCap": "100000", + "initialLeverage": "8", + "notionalCap": "25000", "notionalFloor": "5000", - "maintMarginRatio": "0.05", - "cum": "125.0" + "maintMarginRatio": "0.025", + "cum": "25.0" } }, { "tier": 3.0, "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 6.0, + "info": { + "bracket": "3", + "initialLeverage": "6", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "650.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "3", + "bracket": "4", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5125.0" + "cum": "5650.0" } }, { - "tier": 4.0, + "tier": 5.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "4", + "bracket": "5", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11375.0" + "cum": "11900.0" } }, { - "tier": 5.0, + "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, "maxNotional": 1500000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "5", + "bracket": "6", "initialLeverage": "1", "notionalCap": "1500000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386375.0" + "cum": "386900.0" } } ], @@ -24994,96 +25370,80 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.03, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "20", - "notionalCap": "5000", + "initialLeverage": "10", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.03", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, - "info": { - "bracket": "2", - "initialLeverage": "10", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" - } - }, - { - "tier": 3.0, - "currency": "USDT", "minNotional": 25000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 8.0, "info": { - "bracket": "3", + "bracket": "2", "initialLeverage": "8", "notionalCap": "100000", "notionalFloor": "25000", "maintMarginRatio": "0.05", - "cum": "650.0" + "cum": "500.0" } }, { - "tier": 4.0, + "tier": 3.0, "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5650.0" + "cum": "5500.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11900.0" + "cum": "11750.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 3000000.0, + "maxNotional": 1200000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "1", - "notionalCap": "3000000", + "notionalCap": "1200000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386900.0" + "cum": "386750.0" } } ], @@ -25580,96 +25940,80 @@ "tier": 1.0, "currency": "USDT", "minNotional": 0.0, - "maxNotional": 5000.0, - "maintenanceMarginRate": 0.02, - "maxLeverage": 20.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 10.0, "info": { "bracket": "1", - "initialLeverage": "20", - "notionalCap": "5000", + "initialLeverage": "10", + "notionalCap": "25000", "notionalFloor": "0", - "maintMarginRatio": "0.02", + "maintMarginRatio": "0.025", "cum": "0.0" } }, { "tier": 2.0, "currency": "USDT", - "minNotional": 5000.0, - "maxNotional": 25000.0, - "maintenanceMarginRate": 0.025, - "maxLeverage": 10.0, - "info": { - "bracket": "2", - "initialLeverage": "10", - "notionalCap": "25000", - "notionalFloor": "5000", - "maintMarginRatio": "0.025", - "cum": "25.0" - } - }, - { - "tier": 3.0, - "currency": "USDT", "minNotional": 25000.0, "maxNotional": 100000.0, "maintenanceMarginRate": 0.05, "maxLeverage": 8.0, "info": { - "bracket": "3", + "bracket": "2", "initialLeverage": "8", "notionalCap": "100000", "notionalFloor": "25000", "maintMarginRatio": "0.05", - "cum": "650.0" + "cum": "625.0" } }, { - "tier": 4.0, + "tier": 3.0, "currency": "USDT", "minNotional": 100000.0, "maxNotional": 250000.0, "maintenanceMarginRate": 0.1, "maxLeverage": 5.0, "info": { - "bracket": "4", + "bracket": "3", "initialLeverage": "5", "notionalCap": "250000", "notionalFloor": "100000", "maintMarginRatio": "0.1", - "cum": "5650.0" + "cum": "5625.0" } }, { - "tier": 5.0, + "tier": 4.0, "currency": "USDT", "minNotional": 250000.0, "maxNotional": 1000000.0, "maintenanceMarginRate": 0.125, "maxLeverage": 2.0, "info": { - "bracket": "5", + "bracket": "4", "initialLeverage": "2", "notionalCap": "1000000", "notionalFloor": "250000", "maintMarginRatio": "0.125", - "cum": "11900.0" + "cum": "11875.0" } }, { - "tier": 6.0, + "tier": 5.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 3000000.0, + "maxNotional": 1200000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { - "bracket": "6", + "bracket": "5", "initialLeverage": "1", - "notionalCap": "3000000", + "notionalCap": "1200000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", - "cum": "386900.0" + "cum": "386875.0" } } ], diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 626643d06..187830943 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -1,16 +1,16 @@ """ Bybit exchange subclass """ import logging -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell -from freqtrade.enums import MarginMode, PriceType, TradingMode -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier +from freqtrade.util.datetime_helpers import dt_now, dt_ts logger = logging.getLogger(__name__) @@ -36,6 +36,8 @@ class Bybit(Exchange): "funding_fee_timeframe": "8h", "stoploss_on_exchange": True, "stoploss_order_types": {"limit": "limit", "market": "market"}, + # bybit response parsing fails to populate stopLossPrice + "stop_price_prop": "stopPrice", "stop_price_type_field": "triggerBy", "stop_price_type_value_mapping": { PriceType.LAST: "LastPrice", @@ -203,3 +205,31 @@ class Bybit(Exchange): return self._fetch_and_calculate_funding_fees( pair, amount, is_short, open_date) return 0.0 + + def fetch_orders(self, pair: str, since: datetime, params: Optional[Dict] = None) -> List[Dict]: + """ + Fetch all orders for a pair "since" + :param pair: Pair for the query + :param since: Starting time for the query + """ + # On bybit, the distance between since and "until" can't exceed 7 days. + # we therefore need to split the query into multiple queries. + orders = [] + + while since < dt_now(): + until = since + timedelta(days=7, minutes=-1) + orders += super().fetch_orders(pair, since, params={'until': dt_ts(until)}) + since = until + + return orders + + def fetch_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: + order = super().fetch_order(order_id, pair, params) + if ( + order.get('status') == 'canceled' + and order.get('filled') == 0.0 + and order.get('remaining') == 0.0 + ): + # Canceled orders will have "remaining=0" on bybit. + order['remaining'] = None + return order diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 83a60b906..4592c82a1 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -23,8 +23,7 @@ from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHAN BuySell, Config, EntryExit, ExchangeConfig, ListPairsWithTimeframes, MakerTaker, OBLiteral, PairWithTimeframe) from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list -from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode -from freqtrade.enums.pricetype import PriceType +from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) @@ -62,7 +61,8 @@ class Exchange: # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, - "stop_price_param": "stopPrice", + "stop_price_param": "stopLossPrice", # Used for stoploss_on_exchange request + "stop_price_prop": "stopLossPrice", # Used for stoploss_on_exchange response parsing "order_time_in_force": ["GTC"], "ohlcv_params": {}, "ohlcv_candle_limit": 500, @@ -832,7 +832,7 @@ class Exchange: rate: float, leverage: float, params: Dict = {}, stop_loss: bool = False) -> Dict[str, Any]: now = dt_now() - order_id = f'dry_run_{side}_{now.timestamp()}' + order_id = f'dry_run_{side}_{pair}_{now.timestamp()}' # Rounding here must respect to contract sizes _amount = self._contracts_to_amount( pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount))) @@ -856,15 +856,15 @@ class Exchange: } if stop_loss: dry_order["info"] = {"stopPrice": dry_order["price"]} - dry_order[self._ft_has['stop_price_param']] = dry_order["price"] + dry_order[self._ft_has['stop_price_prop']] = dry_order["price"] # Workaround to avoid filling stoploss orders immediately dry_order["ft_order_type"] = "stoploss" orderbook: Optional[OrderBook] = None if self.exchange_has('fetchL2OrderBook'): orderbook = self.fetch_l2_order_book(pair, 20) if ordertype == "limit" and orderbook: - # Allow a 3% price difference - allowed_diff = 0.03 + # Allow a 1% price difference + allowed_diff = 0.01 if self._dry_is_price_crossed(pair, side, rate, orderbook, allowed_diff): logger.info( f"Converted order {pair} to market order due to price {rate} crossing spread " @@ -920,7 +920,7 @@ class Exchange: max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage)) remaining_amount = amount - filled_amount = 0.0 + filled_value = 0.0 book_entry_price = 0.0 for book_entry in orderbook[ob_type]: book_entry_price = book_entry[0] @@ -928,17 +928,17 @@ class Exchange: if remaining_amount > 0: if remaining_amount < book_entry_coin_volume: # Orderbook at this slot bigger than remaining amount - filled_amount += remaining_amount * book_entry_price + filled_value += remaining_amount * book_entry_price break else: - filled_amount += book_entry_coin_volume * book_entry_price + filled_value += book_entry_coin_volume * book_entry_price remaining_amount -= book_entry_coin_volume else: break else: # If remaining_amount wasn't consumed completely (break was not called) - filled_amount += remaining_amount * book_entry_price - forecast_avg_filled_price = max(filled_amount, 0) / amount + filled_value += remaining_amount * book_entry_price + forecast_avg_filled_price = max(filled_value, 0) / amount # Limit max. slippage to specified value if side == 'buy': forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val) @@ -1008,7 +1008,7 @@ class Exchange: from freqtrade.persistence import Order order = Order.order_by_id(order_id) if order: - ccxt_order = order.to_ccxt_object(self._ft_has['stop_price_param']) + ccxt_order = order.to_ccxt_object(self._ft_has['stop_price_prop']) self._dry_run_open_orders[order_id] = ccxt_order return ccxt_order # Gracefully handle errors with dry-run orders. @@ -1080,6 +1080,13 @@ class Exchange: rate_for_order, params, ) + if order.get('status') is None: + # Map empty status to open. + order['status'] = 'open' + + if order.get('type') is None: + order['type'] = ordertype + self._log_exchange_response('create_order', order) order = self._order_contracts_to_amount(order) return order @@ -1109,7 +1116,7 @@ class Exchange: """ if not self._ft_has.get('stoploss_on_exchange'): raise OperationalException(f"stoploss is not implemented for {self.name}.") - price_param = self._ft_has['stop_price_param'] + price_param = self._ft_has['stop_price_prop'] return ( order.get(price_param, None) is None or ((side == "sell" and stop_loss > float(order[price_param])) or @@ -1421,8 +1428,17 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + orders = [] + if self.exchange_has('fetchClosedOrders'): + orders = self._api.fetch_closed_orders(pair, since=since_ms) + if self.exchange_has('fetchOpenOrders'): + orders_open = self._api.fetch_open_orders(pair, since=since_ms) + orders.extend(orders_open) + return orders + @retrier(retries=0) - def fetch_orders(self, pair: str, since: datetime) -> List[Dict]: + def fetch_orders(self, pair: str, since: datetime, params: Optional[Dict] = None) -> List[Dict]: """ Fetch all orders for a pair "since" :param pair: Pair for the query @@ -1431,26 +1447,20 @@ class Exchange: if self._config['dry_run']: return [] - def fetch_orders_emulate() -> List[Dict]: - orders = [] - if self.exchange_has('fetchClosedOrders'): - orders = self._api.fetch_closed_orders(pair, since=since_ms) - if self.exchange_has('fetchOpenOrders'): - orders_open = self._api.fetch_open_orders(pair, since=since_ms) - orders.extend(orders_open) - return orders - try: since_ms = int((since.timestamp() - 10) * 1000) + if self.exchange_has('fetchOrders'): + if not params: + params = {} try: - orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms) + orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms, params=params) except ccxt.NotSupported: # Some exchanges don't support fetchOrders # attempt to fetch open and closed orders separately - orders = fetch_orders_emulate() + orders = self._fetch_orders_emulate(pair, since_ms) else: - orders = fetch_orders_emulate() + orders = self._fetch_orders_emulate(pair, since_ms) self._log_exchange_response('fetch_orders', orders) orders = [self._order_contracts_to_amount(o) for o in orders] return orders diff --git a/freqtrade/exchange/exchange_utils.py b/freqtrade/exchange/exchange_utils.py index fe7264dd9..4dfa97b28 100644 --- a/freqtrade/exchange/exchange_utils.py +++ b/freqtrade/exchange/exchange_utils.py @@ -248,6 +248,39 @@ def amount_to_contract_precision( return amount +def __price_to_precision_significant_digits( + price: float, + price_precision: float, + *, + rounding_mode: int = ROUND, +) -> float: + """ + Implementation of ROUND_UP/Round_down for significant digits mode. + """ + from decimal import ROUND_DOWN as dec_ROUND_DOWN + from decimal import ROUND_UP as dec_ROUND_UP + from decimal import Decimal + dec = Decimal(str(price)) + string = f'{dec:f}' + precision = round(price_precision) + + q = precision - dec.adjusted() - 1 + sigfig = Decimal('10') ** -q + if q < 0: + string_to_precision = string[:precision] + # string_to_precision is '' when we have zero precision + below = sigfig * Decimal(string_to_precision if string_to_precision else '0') + above = below + sigfig + res = above if rounding_mode == ROUND_UP else below + precise = f'{res:f}' + else: + precise = '{:f}'.format(dec.quantize( + sigfig, + rounding=dec_ROUND_DOWN if rounding_mode == ROUND_DOWN else dec_ROUND_UP) + ) + return float(precise) + + def price_to_precision( price: float, price_precision: Optional[float], @@ -271,28 +304,39 @@ def price_to_precision( :return: price rounded up to the precision the Exchange accepts """ if price_precision is not None and precisionMode is not None: + if rounding_mode not in (ROUND_UP, ROUND_DOWN): + # Use CCXT code where possible. + return float(decimal_to_precision(price, rounding_mode=rounding_mode, + precision=price_precision, + counting_mode=precisionMode + )) + if precisionMode == TICK_SIZE: - if rounding_mode == ROUND: - ticks = price / price_precision - rounded_ticks = round(ticks) - return rounded_ticks * price_precision precision = FtPrecise(price_precision) price_str = FtPrecise(price) missing = price_str % precision if not missing == FtPrecise("0"): - return round(float(str(price_str - missing + precision)), 14) + if rounding_mode == ROUND_UP: + res = price_str - missing + precision + elif rounding_mode == ROUND_DOWN: + res = price_str - missing + return round(float(str(res)), 14) return price - elif precisionMode in (SIGNIFICANT_DIGITS, DECIMAL_PLACES): + elif precisionMode == DECIMAL_PLACES: + ndigits = round(price_precision) - if rounding_mode == ROUND: - return round(price, ndigits) ticks = price * (10**ndigits) if rounding_mode == ROUND_UP: return ceil(ticks) / (10**ndigits) - if rounding_mode == TRUNCATE: - return int(ticks) / (10**ndigits) if rounding_mode == ROUND_DOWN: return floor(ticks) / (10**ndigits) + raise ValueError(f"Unknown rounding_mode {rounding_mode}") + elif precisionMode == SIGNIFICANT_DIGITS: + if rounding_mode in (ROUND_UP, ROUND_DOWN): + return __price_to_precision_significant_digits( + price, price_precision, rounding_mode=rounding_mode + ) + raise ValueError(f"Unknown precisionMode {precisionMode}") return price diff --git a/freqtrade/exchange/gate.py b/freqtrade/exchange/gate.py index d36f57da7..ffef21402 100644 --- a/freqtrade/exchange/gate.py +++ b/freqtrade/exchange/gate.py @@ -25,8 +25,10 @@ class Gate(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, "order_time_in_force": ['GTC', 'IOC'], - "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, + "stoploss_order_types": {"limit": "limit"}, + "stop_price_param": "stopPrice", + "stop_price_prop": "stopPrice", "marketOrderRequiresPrice": True, } diff --git a/freqtrade/exchange/huobi.py b/freqtrade/exchange/huobi.py index fdb6050a3..207520862 100644 --- a/freqtrade/exchange/huobi.py +++ b/freqtrade/exchange/huobi.py @@ -17,6 +17,8 @@ class Huobi(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, + "stop_price_param": "stopPrice", + "stop_price_prop": "stopPrice", "stoploss_order_types": {"limit": "stop-limit"}, "ohlcv_candle_limit": 1000, "l2_limit_range": [5, 10, 20], diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index c41bb6d56..f855ed472 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -24,6 +24,8 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { "stoploss_on_exchange": True, + "stop_price_param": "stopPrice", + "stop_price_prop": "stopPrice", "ohlcv_candle_limit": 720, "ohlcv_has_history": False, "trades_pagination": "id", diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index 20e558513..7033f89ad 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -21,6 +21,8 @@ class Kucoin(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, + "stop_price_param": "stopPrice", + "stop_price_prop": "stopPrice", "stoploss_order_types": {"limit": "limit", "market": "market"}, "l2_limit_range": [20, 100], "l2_limit_range_required": False, diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 7b1c90515..7d7c15f49 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,16 +1,17 @@ import logging +from datetime import timedelta from typing import Any, Dict, List, Optional, Tuple import ccxt from freqtrade.constants import BuySell -from freqtrade.enums import CandleType, MarginMode, TradingMode -from freqtrade.enums.pricetype import PriceType +from freqtrade.enums import CandleType, MarginMode, PriceType, TradingMode from freqtrade.exceptions import (DDosProtection, OperationalException, RetryableOrderError, TemporaryError) from freqtrade.exchange import Exchange, date_minus_candles from freqtrade.exchange.common import retrier from freqtrade.misc import safe_value_fallback2 +from freqtrade.util import dt_now, dt_ts logger = logging.getLogger(__name__) @@ -28,7 +29,6 @@ class Okx(Exchange): "funding_fee_timeframe": "8h", "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, - "stop_price_param": "stopLossPrice", } _ft_has_futures: Dict = { "tickers_have_quoteVolume": False, @@ -187,7 +187,7 @@ class Okx(Exchange): def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: if ( - order['status'] == 'closed' + order.get('status', 'open') == 'closed' and (real_order_id := order.get('info', {}).get('ordId')) is not None ): # Once a order triggered, we fetch the regular followup order. @@ -241,3 +241,18 @@ class Okx(Exchange): pair=pair, params=params1, ) + + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + orders = [] + + orders = self._api.fetch_closed_orders(pair, since=since_ms) + if (since_ms < dt_ts(dt_now() - timedelta(days=6, hours=23))): + # Regular fetch_closed_orders only returns 7 days of data. + # Force usage of "archive" endpoint, which returns 3 months of data. + params = {'method': 'privateGetTradeOrdersHistoryArchive'} + orders_hist = self._api.fetch_closed_orders(pair, since=since_ms, params=params) + orders.extend(orders_hist) + + orders_open = self._api.fetch_open_orders(pair, since=since_ms) + orders.extend(orders_open) + return orders diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index 06ce772a8..5b98efb49 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) torch.multiprocessing.set_sharing_strategy('file_system') SB3_MODELS = ['PPO', 'A2C', 'DQN'] -SB3_CONTRIB_MODELS = ['TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO'] +SB3_CONTRIB_MODELS = ['TRPO', 'ARS', 'RecurrentPPO', 'MaskablePPO', 'QRDQN'] class BaseReinforcementLearningModel(IFreqaiModel): diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b6ded83b1..0306282c0 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -263,23 +263,46 @@ class FreqaiDataDrawer: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() return - def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: + def set_initial_return_values(self, pair: str, + pred_df: DataFrame, + dataframe: DataFrame + ) -> None: """ Set the initial return values to the historical predictions dataframe. This avoids needing to repredict on historical candles, and also stores historical predictions despite retrainings (so stored predictions are true predictions, not just inferencing on trained - data) + data). + + We also aim to keep the date from historical predictions so that the FreqUI displays + zeros during any downtime (between FreqAI reloads). """ - hist_df = self.historic_predictions - len_diff = len(hist_df[pair].index) - len(pred_df.index) - if len_diff < 0: - df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], - ignore_index=True, keys=hist_df[pair].keys()) + new_pred = pred_df.copy() + # set new_pred values to nans (we want to signal to user that there was nothing + # historically made during downtime. The newest pred will get appeneded later in + # append_model_predictions) + new_pred.iloc[:, :] = np.nan + new_pred["date_pred"] = dataframe["date"] + hist_preds = self.historic_predictions[pair].copy() + + # find the closest common date between new_pred and historic predictions + # and cut off the new_pred dataframe at that date + common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner") + if len(common_dates.index) > 0: + new_pred = new_pred.iloc[len(common_dates):] else: - df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + logger.warning("No common dates found between new predictions and historic " + "predictions. You likely left your FreqAI instance offline " + f"for more than {len(dataframe.index)} candles.") + + df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + # remove last row because we will append that later in append_model_predictions() + df_concat = df_concat.iloc[:-1] + # any missing values will get zeroed out so users can see the exact + # downtime in FreqUI df_concat = df_concat.fillna(0) - self.model_return_values[pair] = df_concat + self.historic_predictions[pair] = df_concat + self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: NDArray[np.int_], diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 7d4bf39ca..d58943777 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -244,6 +244,14 @@ class FreqaiDataKitchen: f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points" f" due to NaNs in populated dataset {len(unfiltered_df)}." ) + if len(unfiltered_df) == 0 and not self.live: + raise OperationalException( + f"{self.pair}: all training data dropped due to NaNs. " + "You likely did not download enough training data prior " + "to your backtest timerange. Hint:\n" + f"{DOCS_LINK}/freqai-running/" + "#downloading-data-to-cover-the-full-backtest-period" + ) if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live: worst_indicator = str(unfiltered_df.count().idxmin()) logger.warning( diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index efae6d060..574bec348 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -138,7 +138,6 @@ class IFreqaiModel(ABC): :param metadata: pair metadata coming from strategy. :param strategy: Strategy to train on """ - self.live = strategy.dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE) self.dd.set_pair_dict_info(metadata) self.data_provider = strategy.dp @@ -394,6 +393,11 @@ class IFreqaiModel(ABC): dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only """ + if not strategy.process_only_new_candles: + raise OperationalException("You are trying to use a FreqAI strategy with " + "process_only_new_candles = False. This is not supported " + "by FreqAI, and it is therefore aborting.") + # get the model metadata associated with the current pair (_, trained_timestamp) = self.dd.get_pair_dict_info(metadata["pair"]) @@ -453,7 +457,7 @@ class IFreqaiModel(ABC): pred_df, do_preds = self.predict(dataframe, dk) if pair not in self.dd.historic_predictions: self.set_initial_historic_predictions(pred_df, dk, pair, dataframe) - self.dd.set_initial_return_values(pair, pred_df) + self.dd.set_initial_return_values(pair, pred_df, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) return @@ -645,11 +649,11 @@ class IFreqaiModel(ABC): If the user reuses an identifier on a subsequent instance, this function will not be called. In that case, "real" predictions will be appended to the loaded set of historic predictions. - :param df: DataFrame = the dataframe containing the training feature data - :param model: Any = A model which was `fit` using a common library such as - catboost or lightgbm + :param pred_df: DataFrame = the dataframe containing the predictions coming + out of a model :param dk: FreqaiDataKitchen = object containing methods for data analysis :param pair: str = current pair + :param strat_df: DataFrame = dataframe coming from strategy """ self.dd.historic_predictions[pair] = pred_df diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index beca1f09c..02d43432d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,13 +7,14 @@ from copy import deepcopy from datetime import datetime, time, timedelta, timezone from math import isclose from threading import Lock +from time import sleep from typing import Any, Dict, List, Optional, Tuple from schedule import Scheduler from freqtrade import constants from freqtrade.configuration import validate_config_consistency -from freqtrade.constants import BuySell, Config, ExchangeConfig, LongShort +from freqtrade.constants import BuySell, Config, EntryExecuteMode, ExchangeConfig, LongShort from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge @@ -21,9 +22,8 @@ from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, State, TradingMode) from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) -from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, timeframe_to_minutes, timeframe_to_next_date, - timeframe_to_seconds) -from freqtrade.exchange.common import remove_exchange_credentials +from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, remove_exchange_credentials, + timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds) from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Order, PairLocks, Trade, init_db @@ -373,7 +373,10 @@ class FreqtradeBot(LoggingMixin): "Order is older than 5 days. Assuming order was fully cancelled.") fo = order.to_ccxt_object() fo['status'] = 'canceled' - self.handle_cancel_order(fo, order.trade, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_order( + fo, order.order_id, order.trade, + constants.CANCEL_REASON['TIMEOUT'] + ) except ExchangeError as e: @@ -440,13 +443,6 @@ class FreqtradeBot(LoggingMixin): if fo and fo['status'] == 'open': # Assume this as the open stoploss order trade.stoploss_order_id = order.order_id - elif order.ft_order_side == trade.exit_side: - if fo and fo['status'] == 'open': - # Assume this as the open order - trade.open_order_id = order.order_id - elif order.ft_order_side == trade.entry_side: - if fo and fo['status'] == 'open': - trade.open_order_id = order.order_id if fo: logger.info(f"Found {order} for trade {trade}.") self.update_trade_state(trade, order.order_id, fo, @@ -461,36 +457,48 @@ class FreqtradeBot(LoggingMixin): Only used balance disappeared, which would make exiting impossible. """ try: - orders = self.exchange.fetch_orders(trade.pair, trade.open_date_utc) + orders = self.exchange.fetch_orders( + trade.pair, trade.open_date_utc - timedelta(seconds=10)) + prev_exit_reason = trade.exit_reason + prev_trade_state = trade.is_open for order in orders: trade_order = [o for o in trade.orders if o.order_id == order['id']] - if trade_order: - continue - logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") - order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) - order_obj.order_filled_date = datetime.fromtimestamp( - safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, - tz=timezone.utc) - trade.orders.append(order_obj) - # TODO: how do we handle open_order_id ... - Trade.commit() - prev_exit_reason = trade.exit_reason - trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value - self.update_trade_state(trade, order['id'], order) + if trade_order: + # We knew this order, but didn't have it updated properly + order_obj = trade_order[0] + else: + logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) + order_obj.order_filled_date = datetime.fromtimestamp( + safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, + tz=timezone.utc) + trade.orders.append(order_obj) + Trade.commit() + trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value + + self.update_trade_state(trade, order['id'], order, send_msg=False) logger.info(f"handled order {order['id']}") - if not trade.is_open: - # Trade was just closed - trade.close_date = order_obj.order_filled_date - Trade.commit() - break - else: - trade.exit_reason = prev_exit_reason - Trade.commit() + + # Refresh trade from database + Trade.session.refresh(trade) + if not trade.is_open: + # Trade was just closed + trade.close_date = trade.date_last_filled_utc + self.order_close_notify(trade, order_obj, + order_obj.ft_order_side == 'stoploss', + send_msg=prev_trade_state != trade.is_open) + else: + trade.exit_reason = prev_exit_reason + Trade.commit() except ExchangeError: - logger.warning("Error finding onexchange order") + logger.warning("Error finding onexchange order.") + except Exception: + # catching https://github.com/freqtrade/freqtrade/issues/9025 + logger.warning("Error finding onexchange order", exc_info=True) # # BUY / enter positions / open trades logic and methods # @@ -612,7 +620,8 @@ class FreqtradeBot(LoggingMixin): # Walk through each pair and check if it needs changes for trade in Trade.get_open_trades(): # If there is any open orders, wait for them to finish. - if trade.open_order_id is None: + # TODO Remove to allow mul open orders + if not trade.has_open_orders: # Do a wallets update (will be ratelimited to once per hour) self.wallets.update(False) try: @@ -662,7 +671,7 @@ class FreqtradeBot(LoggingMixin): else: logger.debug("Max adjustment entries is set to unlimited.") self.execute_entry(trade.pair, stake_amount, price=current_entry_rate, - trade=trade, is_short=trade.is_short) + trade=trade, is_short=trade.is_short, mode='pos_adjust') if stake_amount is not None and stake_amount < 0.0: # We should decrease our position @@ -732,7 +741,7 @@ class FreqtradeBot(LoggingMixin): ordertype: Optional[str] = None, enter_tag: Optional[str] = None, trade: Optional[Trade] = None, - order_adjust: bool = False, + mode: EntryExecuteMode = 'initial', leverage_: Optional[float] = None, ) -> bool: """ @@ -749,22 +758,25 @@ class FreqtradeBot(LoggingMixin): pos_adjust = trade is not None enter_limit_requested, stake_amount, leverage = self.get_valid_enter_price_and_stake( - pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust, leverage_, - pos_adjust) + pair, price, stake_amount, trade_side, enter_tag, trade, mode, leverage_) if not stake_amount: return False - msg = (f"Position adjust: about to create a new order for {pair} with stake: " - f"{stake_amount} for {trade}" if pos_adjust + msg = (f"Position adjust: about to create a new order for {pair} with stake_amount: " + f"{stake_amount} for {trade}" if mode == 'pos_adjust' else - f"{name} signal found: about create a new trade for {pair} with stake_amount: " - f"{stake_amount} ...") + (f"Replacing {side} order: about create a new order for {pair} with stake_amount: " + f"{stake_amount} ..." + if mode == 'replace' else + f"{name} signal found: about create a new trade for {pair} with stake_amount: " + f"{stake_amount} ..." + )) logger.info(msg) amount = (stake_amount / enter_limit_requested) * leverage order_type = ordertype or self.strategy.order_types['entry'] - if not pos_adjust and not strategy_safe_wrapper( + if mode == 'initial' and not strategy_safe_wrapper( self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, time_in_force=time_in_force, current_time=datetime.now(timezone.utc), @@ -784,7 +796,7 @@ class FreqtradeBot(LoggingMixin): order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested) order_id = order['id'] order_status = order.get('status') - logger.info(f"Order #{order_id} was created for {pair} and status is {order_status}.") + logger.info(f"Order {order_id} was created for {pair} and status is {order_status}.") # we assume the order is executed at the price requested enter_limit_filled_price = enter_limit_requested @@ -846,7 +858,6 @@ class FreqtradeBot(LoggingMixin): open_rate_requested=enter_limit_requested, open_date=open_date, exchange=self.exchange.id, - open_order_id=order_id, strategy=self.strategy.get_strategy_name(), enter_tag=enter_tag, timeframe=timeframe_to_minutes(self.config['timeframe']), @@ -867,7 +878,6 @@ class FreqtradeBot(LoggingMixin): trade.is_open = True trade.fee_open_currency = None trade.open_rate_requested = enter_limit_requested - trade.open_order_id = order_id trade.orders.append(order_obj) trade.recalc_trade_from_orders() @@ -913,9 +923,8 @@ class FreqtradeBot(LoggingMixin): trade_side: LongShort, entry_tag: Optional[str], trade: Optional[Trade], - order_adjust: bool, + mode: EntryExecuteMode, leverage_: Optional[float], - pos_adjust: bool, ) -> Tuple[float, float, float]: """ Validate and eventually adjust (within limits) limit, amount and leverage @@ -928,11 +937,12 @@ class FreqtradeBot(LoggingMixin): # Calculate price enter_limit_requested = self.exchange.get_rate( pair, side='entry', is_short=(trade_side == 'short'), refresh=True) - if not order_adjust: + if mode != 'replace': # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( - pair=pair, current_time=datetime.now(timezone.utc), + pair=pair, trade=trade, + current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) @@ -967,7 +977,7 @@ class FreqtradeBot(LoggingMixin): # edge-case for now. min_stake_amount = self.exchange.get_min_pair_stake_amount( pair, enter_limit_requested, - self.strategy.stoploss if not pos_adjust else 0.0, + self.strategy.stoploss if not mode != 'pos_adjust' else 0.0, leverage) max_stake_amount = self.exchange.get_max_pair_stake_amount( pair, enter_limit_requested, leverage) @@ -1077,7 +1087,7 @@ class FreqtradeBot(LoggingMixin): trades_closed = 0 for trade in trades: - if trade.open_order_id is None and not self.wallets.check_exit_amount(trade): + if not trade.has_open_orders and not self.wallets.check_exit_amount(trade): logger.warning( f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' 'Trying to recover.') @@ -1095,7 +1105,7 @@ class FreqtradeBot(LoggingMixin): logger.warning( f'Unable to handle stoploss on exchange for {trade.pair}: {exception}') # Check if we can sell our current pair - if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): + if not trade.has_open_orders and trade.is_open and self.handle_trade(trade): trades_closed += 1 except DependencyException as exception: @@ -1214,7 +1224,6 @@ class FreqtradeBot(LoggingMixin): """ logger.debug('Handling stoploss on exchange %s ...', trade) - stoploss_order = None try: @@ -1237,7 +1246,7 @@ class FreqtradeBot(LoggingMixin): self.handle_protections(trade.pair, trade.trade_direction) return True - if trade.open_order_id or not trade.is_open: + if trade.has_open_orders or not trade.is_open: # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case # as the Amount on the exchange is tied up in another trade. # The trade can be closed already (sell-order fill confirmation came in this iteration) @@ -1321,27 +1330,33 @@ class FreqtradeBot(LoggingMixin): Timeout setting takes priority over limit order adjustment request. :return: None """ - for trade in Trade.get_open_order_trades(): - try: - if not trade.open_order_id: + for trade in Trade.get_open_trades(): + for open_order in trade.open_orders: + try: + order = self.exchange.fetch_order(open_order.order_id, trade.pair) + + except (ExchangeError): + logger.info( + 'Cannot query order for %s due to %s', trade, traceback.format_exc() + ) continue - order = self.exchange.fetch_order(trade.open_order_id, trade.pair) - except (ExchangeError): - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue - fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) - not_closed = order['status'] == 'open' or fully_cancelled - order_obj = trade.select_order_by_order_id(trade.open_order_id) + fully_cancelled = self.update_trade_state(trade, open_order.order_id, order) + not_closed = order['status'] == 'open' or fully_cancelled - if not_closed: - if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( - trade, order_obj, datetime.now(timezone.utc))): - self.handle_cancel_order(order, trade, constants.CANCEL_REASON['TIMEOUT']) - else: - self.replace_order(order, order_obj, trade) + if not_closed: + if fully_cancelled or ( + open_order and self.strategy.ft_check_timed_out( + trade, open_order, datetime.now(timezone.utc) + ) + ): + self.handle_cancel_order( + order, open_order.order_id, trade, constants.CANCEL_REASON['TIMEOUT'] + ) + else: + self.replace_order(order, open_order, trade) - def handle_cancel_order(self, order: Dict, trade: Trade, reason: str) -> None: + def handle_cancel_order(self, order: Dict, order_id: str, trade: Trade, reason: str) -> None: """ Check if current analyzed order timed out and cancel if necessary. :param order: Order dict grabbed with exchange.fetch_order() @@ -1349,25 +1364,44 @@ class FreqtradeBot(LoggingMixin): :return: None """ if order['side'] == trade.entry_side: - self.handle_cancel_enter(trade, order, reason) + self.handle_cancel_enter(trade, order, order_id, reason) else: - canceled = self.handle_cancel_exit(trade, order, reason) - canceled_count = trade.get_exit_order_count() + canceled = self.handle_cancel_exit(trade, order, order_id, reason) + canceled_count = trade.get_canceled_exit_order_count() max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) - if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: - logger.warning(f'Emergency exiting trade {trade}, as the exit order ' - f'timed out {max_timeouts} times.') - self.emergency_exit(trade, order['price']) + if (canceled and max_timeouts > 0 and canceled_count >= max_timeouts): + logger.warning(f"Emergency exiting trade {trade}, as the exit order " + f"timed out {max_timeouts} times. force selling {order['amount']}.") + self.emergency_exit(trade, order['price'], order['amount']) - def emergency_exit(self, trade: Trade, price: float) -> None: + def emergency_exit( + self, trade: Trade, price: float, sub_trade_amt: Optional[float] = None) -> None: try: self.execute_trade_exit( trade, price, - exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT)) + exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT), + sub_trade_amt=sub_trade_amt + ) except DependencyException as exception: logger.warning( f'Unable to emergency exit trade {trade.pair}: {exception}') + def replace_order_failed(self, trade: Trade, msg: str) -> None: + """ + Order replacement fail handling. + Deletes the trade if necessary. + :param trade: Trade object. + :param msg: Error message. + """ + logger.warning(msg) + if trade.nr_of_successful_entries == 0: + # this is the first entry and we didn't get filled yet, delete trade + logger.warning(f"Removing {trade} from database.") + self._notify_enter_cancel( + trade, order_type=self.strategy.order_types['entry'], + reason=constants.CANCEL_REASON['REPLACE_FAILED']) + trade.delete() + def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) -> None: """ Check if current analyzed entry order should be replaced or simply cancelled. @@ -1406,19 +1440,24 @@ class FreqtradeBot(LoggingMixin): cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] if order_obj.price != adjusted_entry_price: # cancel existing order if new price is supplied or None - self.handle_cancel_enter(trade, order, cancel_reason, - replacing=replacing) + res = self.handle_cancel_enter(trade, order, order_obj.order_id, cancel_reason, + replacing=replacing) + if not res: + self.replace_order_failed( + trade, f"Could not cancel order for {trade}, therefore not replacing.") + return if adjusted_entry_price: # place new order only if new price is supplied - self.execute_entry( + if not self.execute_entry( pair=trade.pair, stake_amount=( order_obj.safe_remaining * order_obj.safe_price / trade.leverage), price=adjusted_entry_price, trade=trade, is_short=trade.is_short, - order_adjust=True, - ) + mode='replace', + ): + self.replace_order_failed(trade, f"Could not replace order for {trade}.") def cancel_all_open_orders(self) -> None: """ @@ -1426,25 +1465,28 @@ class FreqtradeBot(LoggingMixin): :return: None """ - for trade in Trade.get_open_order_trades(): - if not trade.open_order_id: - continue - try: - order = self.exchange.fetch_order(trade.open_order_id, trade.pair) - except (ExchangeError): - logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) - continue + for trade in Trade.get_open_trades(): + for open_order in trade.open_orders: + try: + order = self.exchange.fetch_order(open_order.order_id, trade.pair) + except (ExchangeError): + logger.info("Can't query order for %s due to %s", trade, traceback.format_exc()) + continue - if order['side'] == trade.entry_side: - self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + if order['side'] == trade.entry_side: + self.handle_cancel_enter( + trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED'] + ) - elif order['side'] == trade.exit_side: - self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + elif order['side'] == trade.exit_side: + self.handle_cancel_exit( + trade, order, open_order.order_id, constants.CANCEL_REASON['ALL_CANCELLED'] + ) Trade.commit() def handle_cancel_enter( - self, trade: Trade, order: Dict, reason: str, - replacing: Optional[bool] = False + self, trade: Trade, order: Dict, order_id: str, + reason: str, replacing: Optional[bool] = False ) -> bool: """ entry cancel - cancel order @@ -1453,11 +1495,10 @@ class FreqtradeBot(LoggingMixin): """ was_trade_fully_canceled = False side = trade.entry_side.capitalize() - if not trade.open_order_id: + if not trade.has_open_orders: logger.warning(f"No open order for {trade}.") return False - # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES: filled_val: float = order.get('filled', 0.0) or 0.0 filled_stake = filled_val * trade.open_rate @@ -1466,16 +1507,27 @@ class FreqtradeBot(LoggingMixin): if filled_val > 0 and minstake and filled_stake < minstake: logger.warning( - f"Order {trade.open_order_id} for {trade.pair} not cancelled, " + f"Order {order_id} for {trade.pair} not cancelled, " f"as the filled amount of {filled_val} would result in an unexitable trade.") return False - corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, + corder = self.exchange.cancel_order_with_result(order_id, trade.pair, trade.amount) + # if replacing, retry fetching the order 3 times if the status is not what we need + if replacing: + retry_count = 0 + while ( + corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES + and retry_count < 3 + ): + sleep(0.5) + corder = self.exchange.fetch_order(order_id, trade.pair) + retry_count += 1 + # Avoid race condition where the order could not be cancelled coz its already filled. # Simply bailing here is the only safe way - as this order will then be # handled in the next iteration. if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES: - logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.") + logger.warning(f"Order {order_id} for {trade.pair} not cancelled.") return False else: # Order was cancelled already, so we can reuse the existing dict @@ -1487,22 +1539,22 @@ class FreqtradeBot(LoggingMixin): # Using filled to determine the filled amount filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled') if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): + was_trade_fully_canceled = True # if trade is not partially completed and it's the only order, just delete the trade - open_order_count = len([order for order in trade.orders if order.status == 'open']) - if open_order_count <= 1 and trade.nr_of_successful_entries == 0 and not replacing: + open_order_count = len([ + order for order in trade.orders if order.ft_is_open and order.order_id != order_id + ]) + if open_order_count < 1 and trade.nr_of_successful_entries == 0 and not replacing: logger.info(f'{side} order fully cancelled. Removing {trade} from database.') trade.delete() - was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" else: - self.update_trade_state(trade, trade.open_order_id, corder) - trade.open_order_id = None + self.update_trade_state(trade, order_id, corder) logger.info(f'{side} Order timeout for {trade}.') else: # update_trade_state (and subsequently recalc_trade_from_orders) will handle updates # to the trade object - self.update_trade_state(trade, trade.open_order_id, corder) - trade.open_order_id = None + self.update_trade_state(trade, order_id, corder) logger.info(f'Partial {trade.entry_side} order timeout for {trade}.') reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" @@ -1512,7 +1564,10 @@ class FreqtradeBot(LoggingMixin): reason=reason) return was_trade_fully_canceled - def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_exit( + self, trade: Trade, order: Dict, order_id: str, + reason: str + ) -> bool: """ exit order cancel - cancel order and update trade :return: True if exit order was cancelled, false otherwise @@ -1520,17 +1575,18 @@ class FreqtradeBot(LoggingMixin): cancelled = False # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES: - filled_val: float = order.get('filled', 0.0) or 0.0 - filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate + filled_amt: float = order.get('filled', 0.0) or 0.0 + # Filled val is in quote currency (after leverage) + filled_rem_stake = trade.stake_amount - (filled_amt * trade.open_rate / trade.leverage) minstake = self.exchange.get_min_pair_stake_amount( trade.pair, trade.open_rate, self.strategy.stoploss) # Double-check remaining amount - if filled_val > 0: + if filled_amt > 0: reason = constants.CANCEL_REASON['PARTIALLY_FILLED'] if minstake and filled_rem_stake < minstake: logger.warning( - f"Order {trade.open_order_id} for {trade.pair} not cancelled, as " - f"the filled amount of {filled_val} would result in an unexitable trade.") + f"Order {order_id} for {trade.pair} not cancelled, as " + f"the filled amount of {filled_amt} would result in an unexitable trade.") reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self._notify_exit_cancel( @@ -1546,7 +1602,7 @@ class FreqtradeBot(LoggingMixin): order['id'], trade.pair, trade.amount) except InvalidOrderException: logger.exception( - f"Could not cancel {trade.exit_side} order {trade.open_order_id}") + f"Could not cancel {trade.exit_side} order {order_id}") return False # Set exit_reason for fill message @@ -1555,14 +1611,12 @@ class FreqtradeBot(LoggingMixin): # Order might be filled above in odd timing issues. if order.get('status') in ('canceled', 'cancelled'): trade.exit_reason = None - trade.open_order_id = None else: trade.exit_reason = exit_reason_prev cancelled = True else: reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] trade.exit_reason = None - trade.open_order_id = None self.update_trade_state(trade, order['id'], order) @@ -1696,7 +1750,6 @@ class FreqtradeBot(LoggingMixin): order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit) trade.orders.append(order_obj) - trade.open_order_id = order['id'] trade.exit_order_status = '' trade.close_rate_requested = limit trade.exit_reason = exit_reason @@ -1704,7 +1757,7 @@ class FreqtradeBot(LoggingMixin): self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): - self.update_trade_state(trade, trade.open_order_id, order) + self.update_trade_state(trade, order_obj.order_id, order) Trade.commit() return True @@ -1723,14 +1776,12 @@ class FreqtradeBot(LoggingMixin): amount = order.safe_filled if fill else order.safe_amount order_rate: float = order.safe_price - profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate) - profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate) + profit = trade.calculate_profit(order_rate, amount, trade.open_rate) else: order_rate = trade.safe_close_rate - profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit) - profit_ratio = trade.calc_profit_ratio(order_rate) + profit = trade.calculate_profit(rate=order_rate) amount = trade.amount - gain = "profit" if profit_ratio > 0 else "loss" + gain = "profit" if profit.profit_ratio > 0 else "loss" msg: RPCSellMsg = { 'type': (RPCMessageType.EXIT_FILL if fill @@ -1748,8 +1799,8 @@ class FreqtradeBot(LoggingMixin): 'open_rate': trade.open_rate, 'close_rate': order_rate, 'current_rate': current_rate, - 'profit_amount': profit, - 'profit_ratio': profit_ratio, + 'profit_amount': profit.profit_abs if fill else profit.total_profit, + 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, 'sell_reason': trade.exit_reason, # Deprecated @@ -1781,11 +1832,10 @@ class FreqtradeBot(LoggingMixin): order = self.order_obj_or_raise(order_id, order_or_none) profit_rate: float = trade.safe_close_rate - profit_trade = trade.calc_profit(rate=profit_rate) + profit = trade.calculate_profit(rate=profit_rate) current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) - profit_ratio = trade.calc_profit_ratio(profit_rate) - gain = "profit" if profit_ratio > 0 else "loss" + gain = "profit" if profit.profit_ratio > 0 else "loss" msg: RPCSellCancelMsg = { 'type': RPCMessageType.EXIT_CANCEL, @@ -1800,8 +1850,8 @@ class FreqtradeBot(LoggingMixin): 'amount': order.safe_amount_after_fee, 'open_rate': trade.open_rate, 'current_rate': current_rate, - 'profit_amount': profit_trade, - 'profit_ratio': profit_ratio, + 'profit_amount': profit.profit_abs, + 'profit_ratio': profit.profit_ratio, 'buy_tag': trade.enter_tag, 'enter_tag': trade.enter_tag, 'sell_reason': trade.exit_reason, # Deprecated @@ -1831,7 +1881,7 @@ class FreqtradeBot(LoggingMixin): def update_trade_state( self, trade: Trade, order_id: Optional[str], - action_order: Optional[Dict[str, Any]] = None, + action_order: Optional[Dict[str, Any]] = None, *, stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary @@ -1868,17 +1918,25 @@ class FreqtradeBot(LoggingMixin): self.handle_order_fee(trade, order_obj, order) - trade.update_trade(order_obj) + trade.update_trade(order_obj, not send_msg) - if order.get('status') in constants.NON_OPEN_EXCHANGE_STATES: + trade = self._update_trade_after_fill(trade, order_obj) + Trade.commit() + + self.order_close_notify(trade, order_obj, stoploss_order, send_msg) + + return False + + def _update_trade_after_fill(self, trade: Trade, order: Order) -> Trade: + if order.status in constants.NON_OPEN_EXCHANGE_STATES: # If a entry order was closed, force update on stoploss on exchange - if order.get('side') == trade.entry_side: + if order.ft_order_side == trade.entry_side: trade = self.cancel_stoploss_on_exchange(trade) if not self.edge: # TODO: should shorting/leverage be supported by Edge, # then this will need to be fixed. trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - if order.get('side') == trade.entry_side or (trade.amount > 0 and trade.is_open): + if order.ft_order_side == trade.entry_side or (trade.amount > 0 and trade.is_open): # Must also run for partial exits # TODO: Margin will need to use interest_rate as well. # interest_rate = self.exchange.get_interest_rate() @@ -1894,13 +1952,16 @@ class FreqtradeBot(LoggingMixin): )) except DependencyException: logger.warning('Unable to calculate liquidation price') + if self.strategy.use_custom_stoploss: + current_rate = self.exchange.get_rate( + trade.pair, side='exit', is_short=trade.is_short, refresh=True) + profit = trade.calc_profit_ratio(current_rate) + self.strategy.ft_stoploss_adjust(current_rate, trade, + datetime.now(timezone.utc), profit, 0, + after_fill=True) # Updating wallets when order is closed self.wallets.update() - Trade.commit() - - self.order_close_notify(trade, order_obj, stoploss_order, send_msg) - - return False + return trade def order_close_notify( self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool): @@ -1910,11 +1971,11 @@ class FreqtradeBot(LoggingMixin): trade.amount, abs_tol=constants.MATH_CLOSE_PREC) if order.ft_order_side == trade.exit_side: # Exit notification - if send_msg and not stoploss_order and not trade.open_order_id: + if send_msg and not stoploss_order and order.order_id not in trade.open_orders_ids: self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order) if not trade.is_open: self.handle_protections(trade.pair, trade.trade_direction) - elif send_msg and not trade.open_order_id and not stoploss_order: + elif send_msg and order.order_id not in trade.open_orders_ids and not stoploss_order: # Enter fill self._notify_enter(trade, order, order.order_type, fill=True, sub_trade=sub_trade) diff --git a/freqtrade/main.py b/freqtrade/main.py index a10620498..05e5409ad 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -11,8 +11,8 @@ from freqtrade.util.gc_setup import gc_set_threshold # check min. python version -if sys.version_info < (3, 8): # pragma: no cover - sys.exit("Freqtrade requires Python version >= 3.8") +if sys.version_info < (3, 9): # pragma: no cover + sys.exit("Freqtrade requires Python version >= 3.9") from freqtrade import __version__ from freqtrade.commands import Arguments diff --git a/freqtrade/misc.py b/freqtrade/misc.py index f8d730fae..cbebf99eb 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -156,7 +156,7 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): +def safe_value_fallback(obj: dict, key1: str, key2: Optional[str] = None, default_value=None): """ Search a value in obj, return this if it's not None. Then search key2 in obj - return that if it's not none - then use default_value. @@ -165,7 +165,7 @@ def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): if key1 in obj and obj[key1] is not None: return obj[key1] else: - if key2 in obj and obj[key2] is not None: + if key2 and key2 in obj and obj[key2] is not None: return obj[key2] return default_value diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 21390489e..ab0ce4e3f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -116,6 +116,7 @@ class Backtesting: raise OperationalException("Timeframe needs to be set in either " "configuration or as cli argument `--timeframe 5m`") self.timeframe = str(self.config.get('timeframe')) + self.disable_database_use() self.timeframe_min = timeframe_to_minutes(self.timeframe) self.init_backtest_detail() self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider) @@ -318,13 +319,16 @@ class Backtesting: else: self.futures_data = {} + def disable_database_use(self): + PairLocks.use_db = False + PairLocks.timeframe = self.timeframe + Trade.use_db = False + def prepare_backtest(self, enable_protections): """ Backtesting setup method - called once for every call to "backtest()". """ - PairLocks.use_db = False - PairLocks.timeframe = self.config['timeframe'] - Trade.use_db = False + self.disable_database_use() PairLocks.reset_locks() Trade.reset_trades() self.rejected_trades = 0 @@ -579,6 +583,11 @@ class Backtesting: """ Rate is within candle, therefore filled""" return row[LOW_IDX] <= rate <= row[HIGH_IDX] + def _call_adjust_stop(self, current_date: datetime, trade: LocalTrade, current_rate: float): + profit = trade.calc_profit_ratio(current_rate) + self.strategy.ft_stoploss_adjust(current_rate, trade, # type: ignore + current_date, profit, 0, after_fill=True) + def _try_close_open_order( self, order: Optional[Order], trade: LocalTrade, current_date: datetime, row: Tuple) -> bool: @@ -588,7 +597,19 @@ class Backtesting: """ if order and self._get_order_filled(order.ft_price, row): order.close_bt_order(current_date, trade) - trade.open_order_id = None + if not (order.ft_order_side == trade.exit_side and order.safe_amount == trade.amount): + # trade is still open + trade.set_liquidation_price(self.exchange.get_liquidation_price( + pair=trade.pair, + open_rate=trade.open_rate, + is_short=trade.is_short, + amount=trade.amount, + stake_amount=trade.stake_amount, + leverage=trade.leverage, + wallet_balance=trade.stake_amount, + )) + self._call_adjust_stop(current_date, trade, order.ft_price) + # pass return True return False @@ -731,7 +752,9 @@ class Backtesting: if order_type == 'limit': new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=propose_rate)( - pair=pair, current_time=current_time, + pair=pair, + trade=trade, # type: ignore[arg-type] + current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate @@ -854,7 +877,6 @@ class Backtesting: self.trade_id_counter += 1 trade = LocalTrade( id=self.trade_id_counter, - open_order_id=self.order_id_counter, pair=pair, base_currency=base_currency, stake_currency=self.config['stake_currency'], @@ -882,16 +904,6 @@ class Backtesting: trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - trade.set_liquidation_price(self.exchange.get_liquidation_price( - pair=pair, - open_rate=propose_rate, - amount=amount, - stake_amount=trade.stake_amount, - leverage=trade.leverage, - wallet_balance=trade.stake_amount, - is_short=is_short, - )) - order = Order( id=self.order_id_counter, ft_trade_id=trade.id, @@ -916,8 +928,7 @@ class Backtesting: ) order._trade_bt = trade trade.orders.append(order) - if not self._try_close_open_order(order, trade, current_time, row): - trade.open_order_id = str(self.order_id_counter) + self._try_close_open_order(order, trade, current_time, row) trade.recalc_trade_from_orders() return trade @@ -929,7 +940,7 @@ class Backtesting: """ for pair in open_trades.keys(): for trade in list(open_trades[pair]): - if trade.open_order_id and trade.nr_of_successful_entries == 0: + if trade.has_open_orders and trade.nr_of_successful_entries == 0: # Ignore trade if entry-order did not fill yet continue exit_row = data[pair][-1] @@ -1006,13 +1017,11 @@ class Backtesting: else: # Close additional entry order del trade.orders[trade.orders.index(order)] - trade.open_order_id = None return False if order.side == trade.exit_side: self.timedout_exit_orders += 1 # Close exit order and retry exiting on next signal. del trade.orders[trade.orders.index(order)] - trade.open_order_id = None return False return None @@ -1040,7 +1049,6 @@ class Backtesting: return False else: del trade.orders[trade.orders.index(order)] - trade.open_order_id = None self.canceled_entry_orders += 1 # place new order if result was not None @@ -1051,7 +1059,7 @@ class Backtesting: order.safe_remaining * order.ft_price / trade.leverage), direction='short' if trade.is_short else 'long') # Delete trade if no successful entries happened (if placing the new order failed) - if trade.open_order_id is None and trade.nr_of_successful_entries == 0: + if not trade.has_open_orders and trade.nr_of_successful_entries == 0: return True self.replaced_entry_orders += 1 else: @@ -1136,7 +1144,7 @@ class Backtesting: self.wallets.update() # 4. Create exit orders (if any) - if not trade.open_order_id: + if not trade.has_open_orders: self._check_trade_exit(trade, row) # Place exit order if necessary # 5. Process exit orders. diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py new file mode 100644 index 000000000..190ac882f --- /dev/null +++ b/freqtrade/optimize/base_analysis.py @@ -0,0 +1,66 @@ +import logging +from copy import deepcopy +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange + + +logger = logging.getLogger(__name__) + + +class VarHolder: + timerange: TimeRange + data: DataFrame + indicators: Dict[str, DataFrame] + result: DataFrame + compared: DataFrame + from_dt: datetime + to_dt: datetime + compared_dt: datetime + timeframe: str + startup_candle: int + + +class BaseAnalysis: + + def __init__(self, config: Dict[str, Any], strategy_obj: Dict): + self.failed_bias_check = True + self.full_varHolder = VarHolder() + self.exchange: Optional[Any] = None + self._fee = None + + # pull variables the scope of the lookahead_analysis-instance + self.local_config = deepcopy(config) + self.local_config['strategy'] = strategy_obj['name'] + self.strategy_obj = strategy_obj + + @staticmethod + def dt_to_timestamp(dt: datetime): + timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) + return timestamp + + def fill_full_varholder(self): + self.full_varHolder = VarHolder() + + # define datetime in human-readable format + parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) + + if parsed_timerange.startdt is None: + self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) + else: + self.full_varHolder.from_dt = parsed_timerange.startdt + + if parsed_timerange.stopdt is None: + self.full_varHolder.to_dt = datetime.utcnow() + else: + self.full_varHolder.to_dt = parsed_timerange.stopdt + + self.prepare_data(self.full_varHolder, self.local_config['pairs']) + + def start(self) -> None: + + # first make a single backtest + self.fill_full_varholder() diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py index f5fe4590e..5beacc6fc 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py @@ -52,7 +52,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): total_profit = sum_daily["profit_ratio_after_slippage"] - minimum_acceptable_return expected_returns_mean = total_profit.mean() - sum_daily['downside_returns'] = 0 + sum_daily['downside_returns'] = 0.0 sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit total_downside = sum_daily['downside_returns'] # Here total_downside contains min(0, P - MAR) values, diff --git a/freqtrade/optimize/lookahead_analysis.py b/freqtrade/optimize/lookahead_analysis.py index 80418da95..924e43e07 100755 --- a/freqtrade/optimize/lookahead_analysis.py +++ b/freqtrade/optimize/lookahead_analysis.py @@ -1,35 +1,23 @@ import logging import shutil from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from pandas import DataFrame -from freqtrade.configuration import TimeRange from freqtrade.data.history import get_timerange from freqtrade.exchange import timeframe_to_minutes from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, restore_verbosity_for_bias_tester) from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder logger = logging.getLogger(__name__) -class VarHolder: - timerange: TimeRange - data: DataFrame - indicators: Dict[str, DataFrame] - result: DataFrame - compared: DataFrame - from_dt: datetime - to_dt: datetime - compared_dt: datetime - timeframe: str - - class Analysis: def __init__(self) -> None: self.total_signals = 0 @@ -39,29 +27,18 @@ class Analysis: self.has_bias = False -class LookaheadAnalysis: +class LookaheadAnalysis(BaseAnalysis): def __init__(self, config: Dict[str, Any], strategy_obj: Dict): - self.failed_bias_check = True - self.full_varHolder = VarHolder() + + super().__init__(config, strategy_obj) self.entry_varHolders: List[VarHolder] = [] self.exit_varHolders: List[VarHolder] = [] - self.exchange: Optional[Any] = None - self._fee = None - # pull variables the scope of the lookahead_analysis-instance - self.local_config = deepcopy(config) - self.local_config['strategy'] = strategy_obj['name'] self.current_analysis = Analysis() self.minimum_trade_amount = config['minimum_trade_amount'] self.targeted_trade_amount = config['targeted_trade_amount'] - self.strategy_obj = strategy_obj - - @staticmethod - def dt_to_timestamp(dt: datetime): - timestamp = int(dt.replace(tzinfo=timezone.utc).timestamp()) - return timestamp @staticmethod def get_result(backtesting: Backtesting, processed: DataFrame): @@ -162,24 +139,6 @@ class LookaheadAnalysis: varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) varholder.result = self.get_result(backtesting, varholder.indicators) - def fill_full_varholder(self): - self.full_varHolder = VarHolder() - - # define datetime in human-readable format - parsed_timerange = TimeRange.parse_timerange(self.local_config['timerange']) - - if parsed_timerange.startdt is None: - self.full_varHolder.from_dt = datetime.fromtimestamp(0, tz=timezone.utc) - else: - self.full_varHolder.from_dt = parsed_timerange.startdt - - if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.utcnow() - else: - self.full_varHolder.to_dt = parsed_timerange.stopdt - - self.prepare_data(self.full_varHolder, self.local_config['pairs']) - def fill_entry_and_exit_varHolders(self, result_row): # entry_varHolder entry_varHolder = VarHolder() @@ -246,8 +205,7 @@ class LookaheadAnalysis: def start(self) -> None: - # first make a single backtest - self.fill_full_varholder() + super().start() reduce_verbosity_for_bias_tester() diff --git a/freqtrade/optimize/lookahead_analysis_helpers.py b/freqtrade/optimize/lookahead_analysis_helpers.py index 422026780..81ea5d61b 100644 --- a/freqtrade/optimize/lookahead_analysis_helpers.py +++ b/freqtrade/optimize/lookahead_analysis_helpers.py @@ -184,12 +184,12 @@ class LookaheadAnalysisSubFunctions: lookaheadAnalysis_instances = [] - # unify --strategy and --strategy_list to one list + # unify --strategy and --strategy-list to one list if not (strategy_list := config.get('strategy_list', [])): if config.get('strategy') is None: raise OperationalException( "No Strategy specified. Please specify a strategy via --strategy or " - "--strategy_list" + "--strategy-list" ) strategy_list = [config['strategy']] @@ -211,5 +211,5 @@ class LookaheadAnalysisSubFunctions: else: logger.error("There were no strategies specified neither through " "--strategy nor through " - "--strategy_list " + "--strategy-list " "or timeframe was not specified.") diff --git a/freqtrade/optimize/recursive_analysis.py b/freqtrade/optimize/recursive_analysis.py new file mode 100644 index 000000000..45c2a457c --- /dev/null +++ b/freqtrade/optimize/recursive_analysis.py @@ -0,0 +1,182 @@ +import logging +import shutil +from copy import deepcopy +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict, List + +from pandas import DataFrame + +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.loggers.set_log_levels import (reduce_verbosity_for_bias_tester, + restore_verbosity_for_bias_tester) +from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.base_analysis import BaseAnalysis, VarHolder + + +logger = logging.getLogger(__name__) + + +class RecursiveAnalysis(BaseAnalysis): + + def __init__(self, config: Dict[str, Any], strategy_obj: Dict): + + self._startup_candle = config.get('startup_candle', [199, 399, 499, 999, 1999]) + + super().__init__(config, strategy_obj) + + self.partial_varHolder_array: List[VarHolder] = [] + self.partial_varHolder_lookahead_array: List[VarHolder] = [] + + self.dict_recursive: Dict[str, Any] = dict() + + # For recursive bias check + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators(self): + + pair_to_check = self.local_config['pairs'][0] + logger.info("Start checking for recursive bias") + + # check and report signals + base_last_row = self.full_varHolder.indicators[pair_to_check].iloc[-1] + + for part in self.partial_varHolder_array: + part_last_row = part.indicators[pair_to_check].iloc[-1] + + compare_df = base_last_row.compare(part_last_row) + if compare_df.shape[0] > 0: + # print(compare_df) + for col_name, values in compare_df.items(): + # print(col_name) + if 'other' == col_name: + continue + indicators = values.index + + for indicator in indicators: + if (indicator not in self.dict_recursive): + self.dict_recursive[indicator] = {} + + values_diff = compare_df.loc[indicator] + values_diff_self = values_diff.loc['self'] + values_diff_other = values_diff.loc['other'] + diff = (values_diff_other - values_diff_self) / values_diff_self * 100 + + self.dict_recursive[indicator][part.startup_candle] = f"{diff:.3f}%" + + else: + logger.info("No difference found. Stop the process.") + break + + # For lookahead bias check + # analyzes two data frames with processed indicators and shows differences between them. + def analyze_indicators_lookahead(self): + + pair_to_check = self.local_config['pairs'][0] + logger.info("Start checking for lookahead bias on indicators only") + + part = self.partial_varHolder_lookahead_array[0] + part_last_row = part.indicators[pair_to_check].iloc[-1] + date_to_check = part_last_row['date'] + index_to_get = (self.full_varHolder.indicators[pair_to_check]['date'] == date_to_check) + base_row_check = self.full_varHolder.indicators[pair_to_check].loc[index_to_get].iloc[-1] + + check_time = part.to_dt.strftime('%Y-%m-%dT%H:%M:%S') + + logger.info(f"Check indicators at {check_time}") + # logger.info(f"vs {part_timerange} with {part.startup_candle} startup candle") + + compare_df = base_row_check.compare(part_last_row) + if compare_df.shape[0] > 0: + # print(compare_df) + for col_name, values in compare_df.items(): + # print(col_name) + if 'other' == col_name: + continue + indicators = values.index + + for indicator in indicators: + logger.info(f"=> found lookahead in indicator {indicator}") + # logger.info("base value {:.5f}".format(values_diff_self)) + # logger.info("part value {:.5f}".format(values_diff_other)) + + else: + logger.info("No lookahead bias on indicators found. Stop the process.") + + def prepare_data(self, varholder: VarHolder, pairs_to_load: List[DataFrame]): + + if 'freqai' in self.local_config and 'identifier' in self.local_config['freqai']: + # purge previous data if the freqai model is defined + # (to be sure nothing is carried over from older backtests) + path_to_current_identifier = ( + Path(f"{self.local_config['user_data_dir']}/models/" + f"{self.local_config['freqai']['identifier']}").resolve()) + # remove folder and its contents + if Path.exists(path_to_current_identifier): + shutil.rmtree(path_to_current_identifier) + + prepare_data_config = deepcopy(self.local_config) + prepare_data_config['timerange'] = (str(self.dt_to_timestamp(varholder.from_dt)) + "-" + + str(self.dt_to_timestamp(varholder.to_dt))) + prepare_data_config['exchange']['pair_whitelist'] = pairs_to_load + + backtesting = Backtesting(prepare_data_config, self.exchange) + backtesting._set_strategy(backtesting.strategylist[0]) + + varholder.data, varholder.timerange = backtesting.load_bt_data() + backtesting.load_bt_data_detail() + varholder.timeframe = backtesting.timeframe + + varholder.indicators = backtesting.strategy.advise_all_indicators(varholder.data) + + def fill_partial_varholder(self, start_date, startup_candle): + logger.info(f"Calculating indicators using startup candle of {startup_candle}.") + partial_varHolder = VarHolder() + + partial_varHolder.from_dt = start_date + partial_varHolder.to_dt = self.full_varHolder.to_dt + partial_varHolder.startup_candle = startup_candle + + self.local_config['startup_candle_count'] = startup_candle + + self.prepare_data(partial_varHolder, self.local_config['pairs']) + + self.partial_varHolder_array.append(partial_varHolder) + + def fill_partial_varholder_lookahead(self, end_date): + logger.info("Calculating indicators to test lookahead on indicators.") + + partial_varHolder = VarHolder() + + partial_varHolder.from_dt = self.full_varHolder.from_dt + partial_varHolder.to_dt = end_date + + self.prepare_data(partial_varHolder, self.local_config['pairs']) + + self.partial_varHolder_lookahead_array.append(partial_varHolder) + + def start(self) -> None: + + super().start() + + reduce_verbosity_for_bias_tester() + start_date_full = self.full_varHolder.from_dt + end_date_full = self.full_varHolder.to_dt + + timeframe_minutes = timeframe_to_minutes(self.full_varHolder.timeframe) + + end_date_partial = start_date_full + timedelta(minutes=int(timeframe_minutes * 10)) + + self.fill_partial_varholder_lookahead(end_date_partial) + + # restore_verbosity_for_bias_tester() + + start_date_partial = end_date_full - timedelta(minutes=int(timeframe_minutes)) + + for startup_candle in self._startup_candle: + self.fill_partial_varholder(start_date_partial, int(startup_candle)) + + # Restore verbosity, so it's not too quiet for the next strategy + restore_verbosity_for_bias_tester() + + self.analyze_indicators() + self.analyze_indicators_lookahead() diff --git a/freqtrade/optimize/recursive_analysis_helpers.py b/freqtrade/optimize/recursive_analysis_helpers.py new file mode 100644 index 000000000..ba1cf8745 --- /dev/null +++ b/freqtrade/optimize/recursive_analysis_helpers.py @@ -0,0 +1,106 @@ +import logging +import time +from pathlib import Path +from typing import Any, Dict, List + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.recursive_analysis import RecursiveAnalysis +from freqtrade.resolvers import StrategyResolver + + +logger = logging.getLogger(__name__) + + +class RecursiveAnalysisSubFunctions: + + @staticmethod + def text_table_recursive_analysis_instances( + recursive_instances: List[RecursiveAnalysis]): + startups = recursive_instances[0]._startup_candle + headers = ['indicators'] + for candle in startups: + headers.append(candle) + + data = [] + for inst in recursive_instances: + if len(inst.dict_recursive) > 0: + for indicator, values in inst.dict_recursive.items(): + temp_data = [indicator] + for candle in startups: + temp_data.append(values.get(int(candle), '-')) + data.append(temp_data) + + from tabulate import tabulate + table = tabulate(data, headers=headers, tablefmt="orgtbl") + print(table) + return table, headers, data + + @staticmethod + def calculate_config_overrides(config: Config): + if 'timerange' not in config: + # setting a timerange is enforced here + raise OperationalException( + "Please set a timerange. " + "A timerange of 5000 candles are enough for recursive analysis." + ) + + if config.get('backtest_cache') is None: + config['backtest_cache'] = 'none' + elif config['backtest_cache'] != 'none': + logger.info(f"backtest_cache = " + f"{config['backtest_cache']} detected. " + f"Inside recursive-analysis it is enforced to be 'none'. " + f"Changed it to 'none'") + config['backtest_cache'] = 'none' + return config + + @staticmethod + def initialize_single_recursive_analysis(config: Config, strategy_obj: Dict[str, Any]): + + logger.info(f"Recursive test of {Path(strategy_obj['location']).name} started.") + start = time.perf_counter() + current_instance = RecursiveAnalysis(config, strategy_obj) + current_instance.start() + elapsed = time.perf_counter() - start + logger.info(f"Checking recursive and indicator-only lookahead bias of indicators " + f"of {Path(strategy_obj['location']).name} " + f"took {elapsed:.0f} seconds.") + return current_instance + + @staticmethod + def start(config: Config): + config = RecursiveAnalysisSubFunctions.calculate_config_overrides(config) + + strategy_objs = StrategyResolver.search_all_objects( + config, enum_failed=False, recursive=config.get('recursive_strategy_search', False)) + + RecursiveAnalysis_instances = [] + + # unify --strategy and --strategy-list to one list + if not (strategy_list := config.get('strategy_list', [])): + if config.get('strategy') is None: + raise OperationalException( + "No Strategy specified. Please specify a strategy via --strategy or " + "--strategy-list" + ) + strategy_list = [config['strategy']] + + # check if strategies can be properly loaded, only check them if they can be. + for strat in strategy_list: + for strategy_obj in strategy_objs: + if strategy_obj['name'] == strat and strategy_obj not in strategy_list: + RecursiveAnalysis_instances.append( + RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis( + config, strategy_obj)) + break + + # report the results + if RecursiveAnalysis_instances: + RecursiveAnalysisSubFunctions.text_table_recursive_analysis_instances( + RecursiveAnalysis_instances) + else: + logger.error("There were no strategies specified neither through " + "--strategy nor through " + "--strategy-list " + "or timeframe was not specified.") diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 87b172846..69d37530f 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -88,6 +88,9 @@ def migrate_trades_and_orders_table( stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + is_stop_loss_trailing = get_column_def( + cols, 'is_stop_loss_trailing', + f'coalesce({stop_loss_pct}, 0.0) <> coalesce({initial_stop_loss_pct}, 0.0)') stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') max_rate = get_column_def(cols, 'max_rate', '0.0') @@ -154,9 +157,9 @@ def migrate_trades_and_orders_table( fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_close_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stake_amount, amount, amount_requested, open_date, close_date, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, + is_stop_loss_trailing, stoploss_order_id, stoploss_last_update, max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag, timeframe, open_trade_value, close_profit_abs, trading_mode, leverage, liquidation_price, is_short, @@ -171,10 +174,11 @@ def migrate_trades_and_orders_table( {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + stake_amount, amount, {amount_requested}, open_date, close_date, {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, {initial_stop_loss} initial_stop_loss, {initial_stop_loss_pct} initial_stop_loss_pct, + {is_stop_loss_trailing} is_stop_loss_trailing, {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, {max_rate} max_rate, {min_rate} min_rate, case when {exit_reason} = 'sell_signal' then 'exit_signal' @@ -268,6 +272,13 @@ def set_sqlite_to_wal(engine): def fix_old_dry_orders(engine): with engine.begin() as connection: + + # Update current dry-run Orders where + # - current Order is open + # - current Trade is closed + # - current Order trade_id not equal to current Trade.id + # - current Order not stoploss + stmt = update(Order).where( Order.ft_is_open.is_(True), tuple_(Order.ft_trade_id, Order.order_id).not_in( @@ -281,12 +292,13 @@ def fix_old_dry_orders(engine): ).values(ft_is_open=False) connection.execute(stmt) + # Close dry-run orders for closed trades. stmt = update(Order).where( Order.ft_is_open.is_(True), - tuple_(Order.ft_trade_id, Order.order_id).not_in( + Order.ft_trade_id.not_in( select( - Trade.id, Trade.open_order_id - ).where(Trade.open_order_id.is_not(None)) + Trade.id + ).where(Trade.is_open.is_(True)) ), Order.ft_order_side != 'stoploss', Order.order_id.like('dry%') @@ -316,8 +328,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None: # if ('orders' not in previous_tables # or not has_column(cols_orders, 'funding_fee')): migrating = False - # if not has_column(cols_trades, 'max_stake_amount'): - if not has_column(cols_orders, 'ft_price'): + # if not has_column(cols_orders, 'ft_price'): + if not has_column(cols_trades, 'is_stop_loss_trailing'): migrating = True logger.info(f"Running database migration for trades - " f"backup: {table_back_name}, {order_table_bak_name}") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 275cccaf8..8efce76d1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -3,6 +3,7 @@ This module contains the class to persist trades into SQLite """ import logging from collections import defaultdict +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from math import isclose from typing import Any, ClassVar, Dict, List, Optional, Sequence, cast @@ -12,20 +13,30 @@ from sqlalchemy import (Enum, Float, ForeignKey, Integer, ScalarResult, Select, from sqlalchemy.orm import Mapped, lazyload, mapped_column, relationship, validates from typing_extensions import Self -from freqtrade.constants import (CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, - NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) +from freqtrade.constants import (CANCELED_EXCHANGE_STATES, CUSTOM_TAG_MAX_LENGTH, + DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, + BuySell, LongShort) from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precision, price_to_precision) from freqtrade.leverage import interest +from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise, dt_now +from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts logger = logging.getLogger(__name__) +@dataclass +class ProfitStruct: + profit_abs: float + profit_ratio: float + total_profit: float + total_profit_ratio: float + + class Order(ModelBase): """ Order database model @@ -167,7 +178,9 @@ class Order(ModelBase): # (represents the funding fee since the last order) self.funding_fee = self.trade.funding_fees if (order.get('filled', 0.0) or 0.0) > 0 and not self.order_filled_date: - self.order_filled_date = datetime.now(timezone.utc) + self.order_filled_date = dt_from_ts( + safe_value_fallback(order, 'lastTradeTimestamp', default_value=dt_ts()) + ) self.order_update_date = datetime.now(timezone.utc) def to_ccxt_object(self, stopPriceName: str = 'stopPrice') -> Dict[str, Any]: @@ -240,7 +253,10 @@ class Order(ModelBase): if (self.ft_order_side == trade.entry_side and self.price): trade.open_rate = self.price trade.recalc_trade_from_orders() - trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True) + if trade.nr_of_successful_entries == 1: + trade.initial_stop_loss_pct = None + trade.is_stop_loss_trailing = False + trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct) @staticmethod def update_orders(orders: List['Order'], order: Dict[str, Any]): @@ -340,7 +356,6 @@ class LocalTrade: amount_requested: Optional[float] = None open_date: datetime close_date: Optional[datetime] = None - open_order_id: Optional[str] = None # absolute value of the stop loss stop_loss: float = 0.0 # percentage value of the stop loss @@ -349,6 +364,7 @@ class LocalTrade: initial_stop_loss: Optional[float] = 0.0 # percentage value of the initial stop loss initial_stop_loss_pct: Optional[float] = None + is_stop_loss_trailing: bool = False # stoploss order id which is on exchange stoploss_order_id: Optional[str] = None # last update time of the stoploss order on exchange @@ -418,13 +434,20 @@ class LocalTrade: return self.amount @property - def date_last_filled_utc(self) -> datetime: + def _date_last_filled_utc(self) -> Optional[datetime]: """ Date of the last filled order""" orders = self.select_filled_orders() - if not orders: + if orders: + return max(o.order_filled_utc for o in orders if o.order_filled_utc) + return None + + @property + def date_last_filled_utc(self) -> datetime: + """ Date of the last filled order - or open_date if no orders are filled""" + dt_last_filled = self._date_last_filled_utc + if not dt_last_filled: return self.open_date_utc - return max([self.open_date_utc, - max(o.order_filled_utc for o in orders if o.order_filled_utc)]) + return max([self.open_date_utc, dt_last_filled]) @property def open_date_utc(self): @@ -481,6 +504,32 @@ class LocalTrade: except IndexError: return '' + @property + def open_orders(self) -> List[Order]: + """ + All open orders for this trade excluding stoploss orders + """ + return [o for o in self.orders if o.ft_is_open and o.ft_order_side != 'stoploss'] + + @property + def has_open_orders(self) -> int: + """ + True if there are open orders for this trade excluding stoploss orders + """ + open_orders_wo_sl = [ + o for o in self.orders + if o.ft_order_side not in ['stoploss'] and o.ft_is_open + ] + return len(open_orders_wo_sl) > 0 + + @property + def open_orders_ids(self) -> List[str]: + open_orders_ids_wo_sl = [ + oo.order_id for oo in self.open_orders + if oo.ft_order_side not in ['stoploss'] + ] + return open_orders_ids_wo_sl + def __init__(self, **kwargs): for key in kwargs: setattr(self, key, kwargs[key]) @@ -499,8 +548,8 @@ class LocalTrade: ) def to_json(self, minified: bool = False) -> Dict[str, Any]: - filled_orders = self.select_filled_or_open_orders() - orders = [order.to_json(self.entry_side, minified) for order in filled_orders] + filled_or_open_orders = self.select_filled_or_open_orders() + orders_json = [order.to_json(self.entry_side, minified) for order in filled_or_open_orders] return { 'trade_id': self.id, @@ -576,11 +625,12 @@ class LocalTrade: 'is_short': self.is_short, 'trading_mode': self.trading_mode, 'funding_fees': self.funding_fees, - 'open_order_id': self.open_order_id, 'amount_precision': self.amount_precision, 'price_precision': self.price_precision, 'precision_mode': self.precision_mode, - 'orders': orders, + 'contract_size': self.contract_size, + 'has_open_orders': self.has_open_orders, + 'orders': orders_json, } @staticmethod @@ -621,18 +671,18 @@ class LocalTrade: self.stop_loss_pct = -1 * abs(percent) def adjust_stop_loss(self, current_price: float, stoploss: Optional[float], - initial: bool = False, refresh: bool = False) -> None: + initial: bool = False, allow_refresh: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting :param current_price: Current rate the asset is traded :param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price). :param initial: Called to initiate stop_loss. Skips everything if self.stop_loss is already set. + :param refresh: Called to refresh stop_loss, allows adjustment in both directions """ if stoploss is None or (initial and not (self.stop_loss is None or self.stop_loss == 0)): # Don't modify if called with initial and nothing to do return - refresh = True if refresh and self.nr_of_successful_entries == 1 else False leverage = self.leverage or 1.0 if self.is_short: @@ -643,7 +693,7 @@ class LocalTrade: stop_loss_norm = price_to_precision(new_loss, self.price_precision, self.precision_mode, rounding_mode=ROUND_DOWN if self.is_short else ROUND_UP) # no stop loss assigned yet - if self.initial_stop_loss_pct is None or refresh: + if self.initial_stop_loss_pct is None: self.__set_stop_loss(stop_loss_norm, stoploss) self.initial_stop_loss = price_to_precision( stop_loss_norm, self.price_precision, self.precision_mode, @@ -658,8 +708,14 @@ class LocalTrade: # stop losses only walk up, never down!, # ? But adding more to a leveraged trade would create a lower liquidation price, # ? decreasing the minimum stoploss - if (higher_stop and not self.is_short) or (lower_stop and self.is_short): + if ( + allow_refresh + or (higher_stop and not self.is_short) + or (lower_stop and self.is_short) + ): logger.debug(f"{self.pair} - Adjusting stoploss...") + if not allow_refresh: + self.is_stop_loss_trailing = True self.__set_stop_loss(stop_loss_norm, stoploss) else: logger.debug(f"{self.pair} - Keeping current stoploss...") @@ -672,7 +728,7 @@ class LocalTrade: f"Trailing stoploss saved us: " f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.") - def update_trade(self, order: Order) -> None: + def update_trade(self, order: Order, recalculating: bool = False) -> None: """ Updates this entity with amount and actual open/close rates. :param order: order retrieved by exchange.fetch_order() @@ -692,24 +748,13 @@ class LocalTrade: if self.is_open: payment = "SELL" if self.is_short else "BUY" logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - # condition to avoid reset value when updating fees - if self.open_order_id == order.order_id: - self.open_order_id = None - else: - logger.warning( - f'Got different open_order_id {self.open_order_id} != {order.order_id}') + self.recalc_trade_from_orders() elif order.ft_order_side == self.exit_side: if self.is_open: payment = "BUY" if self.is_short else "SELL" # * On margin shorts, you buy a little bit more than the amount (amount + interest) logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.') - # condition to avoid reset value when updating fees - if self.open_order_id == order.order_id: - self.open_order_id = None - else: - logger.warning( - f'Got different open_order_id {self.open_order_id} != {order.order_id}') elif order.ft_order_side == 'stoploss' and order.status not in ('open', ): self.stoploss_order_id = None @@ -725,8 +770,9 @@ class LocalTrade: self.precision_mode, self.contract_size) if ( isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC) - or order.safe_amount_after_fee > amount_tr + or (not recalculating and order.safe_amount_after_fee > amount_tr) ): + # When recalculating a trade, only comming out to 0 can force a close self.close(order.safe_price) else: self.recalc_trade_from_orders() @@ -739,10 +785,9 @@ class LocalTrade: and marks trade as closed """ self.close_rate = rate - self.close_date = self.close_date or datetime.utcnow() + self.close_date = self.close_date or self._date_last_filled_utc or dt_now() self.is_open = False self.exit_order_status = 'closed' - self.open_order_id = None self.recalc_trade_from_orders(is_closing=True) if show_msg: logger.info(f"Marking {self} as closed as the trade is fulfilled " @@ -780,12 +825,13 @@ class LocalTrade: def update_order(self, order: Dict) -> None: Order.update_orders(self.orders, order) - def get_exit_order_count(self) -> int: + def get_canceled_exit_order_count(self) -> int: """ Get amount of failed exiting orders assumes full exits. """ - return len([o for o in self.orders if o.ft_order_side == self.exit_side]) + return len([o for o in self.orders if o.ft_order_side == self.exit_side + and o.status in CANCELED_EXCHANGE_STATES]) def _calc_open_trade_value(self, amount: float, open_rate: float) -> float: """ @@ -878,11 +924,26 @@ class LocalTrade: open_rate: Optional[float] = None) -> float: """ Calculate the absolute profit in stake currency between Close and Open trade + Deprecated - only available for backwards compatibility :param rate: close rate to compare with. :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. :return: profit in stake currency as float """ + prof = self.calculate_profit(rate, amount, open_rate) + return prof.profit_abs + + def calculate_profit(self, rate: float, amount: Optional[float] = None, + open_rate: Optional[float] = None) -> ProfitStruct: + """ + Calculate profit metrics (absolute, ratio, total, total ratio). + All calculations include fees. + :param rate: close rate to compare with. + :param amount: Amount to use for the calculation. Falls back to trade.amount if not set. + :param open_rate: open_rate to use. Defaults to self.open_rate if not provided. + :return: Profit structure, containing absolute and relative profits. + """ + close_trade_value = self.calc_close_trade_value(rate, amount) if amount is None or open_rate is None: open_trade_value = self.open_trade_value @@ -890,10 +951,33 @@ class LocalTrade: open_trade_value = self._calc_open_trade_value(amount, open_rate) if self.is_short: - profit = open_trade_value - close_trade_value + profit_abs = open_trade_value - close_trade_value else: - profit = close_trade_value - open_trade_value - return float(f"{profit:.8f}") + profit_abs = close_trade_value - open_trade_value + + try: + if self.is_short: + profit_ratio = (1 - (close_trade_value / open_trade_value)) * self.leverage + else: + profit_ratio = ((close_trade_value / open_trade_value) - 1) * self.leverage + profit_ratio = float(f"{profit_ratio:.8f}") + except ZeroDivisionError: + profit_ratio = 0.0 + + total_profit_abs = profit_abs + self.realized_profit + total_profit_ratio = ( + (total_profit_abs / self.max_stake_amount) * self.leverage + if self.max_stake_amount else 0.0 + ) + total_profit_ratio = float(f"{total_profit_ratio:.8f}") + profit_abs = float(f"{profit_abs:.8f}") + + return ProfitStruct( + profit_abs=profit_abs, + profit_ratio=profit_ratio, + total_profit=profit_abs + self.realized_profit, + total_profit_ratio=total_profit_ratio, + ) def calc_profit_ratio( self, rate: float, amount: Optional[float] = None, @@ -914,15 +998,14 @@ class LocalTrade: short_close_zero = (self.is_short and close_trade_value == 0.0) long_close_zero = (not self.is_short and open_trade_value == 0.0) - leverage = self.leverage or 1.0 if (short_close_zero or long_close_zero): return 0.0 else: if self.is_short: - profit_ratio = (1 - (close_trade_value / open_trade_value)) * leverage + profit_ratio = (1 - (close_trade_value / open_trade_value)) * self.leverage else: - profit_ratio = ((close_trade_value / open_trade_value) - 1) * leverage + profit_ratio = ((close_trade_value / open_trade_value) - 1) * self.leverage return float(f"{profit_ratio:.8f}") @@ -935,7 +1018,6 @@ class LocalTrade: avg_price = FtPrecise(0.0) close_profit = 0.0 close_profit_abs = 0.0 - profit = None # Reset funding fees self.funding_fees = 0.0 funding_fees = 0.0 @@ -965,11 +1047,9 @@ class LocalTrade: exit_rate = o.safe_price exit_amount = o.safe_amount_after_fee - profit = self.calc_profit(rate=exit_rate, amount=exit_amount, - open_rate=float(avg_price)) - close_profit_abs += profit - close_profit = self.calc_profit_ratio( - exit_rate, amount=exit_amount, open_rate=avg_price) + prof = self.calculate_profit(exit_rate, exit_amount, float(avg_price)) + close_profit_abs += prof.profit_abs + close_profit = prof.profit_ratio else: total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price) max_stake_amount += (tmp_amount * price) @@ -979,7 +1059,7 @@ class LocalTrade: if close_profit: self.close_profit = close_profit self.realized_profit = close_profit_abs - self.close_profit_abs = profit + self.close_profit_abs = prof.profit_abs current_amount_tr = amount_to_contract_precision( float(current_amount), self.amount_precision, self.precision_mode, self.contract_size) @@ -1194,7 +1274,7 @@ class LocalTrade: logger.info(f"Found open trade: {trade}") # skip case if trailing-stop changed the stoploss already. - if (trade.stop_loss == trade.initial_stop_loss + if (not trade.is_stop_loss_trailing and trade.initial_stop_loss_pct != desired_stoploss): # Stoploss value got changed @@ -1256,7 +1336,6 @@ class Trade(ModelBase, LocalTrade): open_date: Mapped[datetime] = mapped_column( nullable=False, default=datetime.utcnow) # type: ignore close_date: Mapped[Optional[datetime]] = mapped_column() # type: ignore - open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # type: ignore # absolute value of the stop loss stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0) # type: ignore # percentage value of the stop loss @@ -1267,6 +1346,8 @@ class Trade(ModelBase, LocalTrade): # percentage value of the initial stop loss initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column( Float(), nullable=True) # type: ignore + is_stop_loss_trailing: Mapped[bool] = mapped_column( + nullable=False, default=False) # type: ignore # stoploss order id which is on exchange stoploss_order_id: Mapped[Optional[str]] = mapped_column( String(255), nullable=True, index=True) # type: ignore @@ -1411,14 +1492,6 @@ class Trade(ModelBase, LocalTrade): # raise an exception. return Trade.session.scalars(query) - @staticmethod - def get_open_order_trades() -> List['Trade']: - """ - Returns all open trades - NOTE: Not supported in Backtesting. - """ - return cast(List[Trade], Trade.get_trades(Trade.open_order_id.isnot(None)).all()) - @staticmethod def get_open_trades_without_assigned_fees(): """ @@ -1717,7 +1790,10 @@ class Trade(ModelBase, LocalTrade): is_short=data["is_short"], trading_mode=data["trading_mode"], funding_fees=data["funding_fees"], - open_order_id=data["open_order_id"], + amount_precision=data.get('amount_precision', None), + price_precision=data.get('price_precision', None), + precision_mode=data.get('precision_mode', None), + contract_size=data.get('contract_size', None), ) for order in data["orders"]: diff --git a/freqtrade/plugins/pairlist/FullTradesFilter.py b/freqtrade/plugins/pairlist/FullTradesFilter.py new file mode 100644 index 000000000..69779d896 --- /dev/null +++ b/freqtrade/plugins/pairlist/FullTradesFilter.py @@ -0,0 +1,57 @@ +""" +Full trade slots pair list filter +""" +import logging +from typing import Any, Dict, List + +from freqtrade.constants import Config +from freqtrade.exchange.types import Tickers +from freqtrade.persistence import Trade +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class FullTradesFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short allowlist method description - used for startup-messages + """ + return f"{self.name} - Shrink whitelist when trade slots are full." + + @staticmethod + def description() -> str: + return "Shrink whitelist when trade slots are full." + + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: + """ + Filters and sorts pairlist and returns the allowlist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new allowlist + """ + # Get the number of open trades and max open trades config + num_open = Trade.get_open_trade_count() + max_trades = self._config['max_open_trades'] + + if (num_open >= max_trades) and (max_trades > 0): + return [] + + return pairlist diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 9e4a4fca9..c2f102777 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -260,6 +260,7 @@ class VolumePairList(IPairList): quoteVolume = (pair_candles['quoteVolume'] .rolling(self._lookback_period) .sum() + .fillna(0) .iloc[-1]) # replace quoteVolume with range quoteVolume sum calculated above diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 93d4fc308..ca8fdc200 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -29,9 +29,8 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], except re.error as err: raise ValueError(f"Wildcard error in {pair_wc}, {err}") - for element in result: - if not re.fullmatch(r'^[A-Za-z0-9/-]+$', element): - result.remove(element) + result = [element for element in result if re.fullmatch(r'^[A-Za-z0-9:/-]+$', element)] + else: for pair_wc in wildcardpl: try: diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f294b882b..f4625f572 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -30,7 +30,7 @@ class RangeStabilityFilter(IPairList): self._days = pairlistconfig.get('lookback_days', 10) self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') - self._refresh_period = pairlistconfig.get('refresh_period', 1440) + self._refresh_period = pairlistconfig.get('refresh_period', 86400) self._def_candletype = self._config['candle_type_def'] self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 6f5b6655d..7e0204c0e 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -218,6 +218,12 @@ class StrategyResolver(IResolver): "Please update your strategy to implement " "`populate_indicators`, `populate_entry_trend` and `populate_exit_trend` " "with the metadata argument. ") + + has_after_fill = ('after_fill' in getfullargspec(strategy.custom_stoploss).args + and check_override(strategy, IStrategy, 'custom_stoploss')) + if has_after_fill: + strategy._ft_stop_uses_after_fill = True + return strategy @staticmethod diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 892865d43..97f6251bc 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -141,6 +141,10 @@ class Profit(BaseModel): expectancy_ratio: float max_drawdown: float max_drawdown_abs: float + max_drawdown_start: str + max_drawdown_start_timestamp: int + max_drawdown_end: str + max_drawdown_end_timestamp: int trading_volume: Optional[float] = None bot_start_timestamp: int bot_start_date: str @@ -157,7 +161,7 @@ class Stats(BaseModel): durations: Dict[str, Optional[float]] -class DailyRecord(BaseModel): +class DailyWeeklyMonthlyRecord(BaseModel): date: date abs_profit: float rel_profit: float @@ -166,8 +170,8 @@ class DailyRecord(BaseModel): trade_count: int -class Daily(BaseModel): - data: List[DailyRecord] +class DailyWeeklyMonthly(BaseModel): + data: List[DailyWeeklyMonthlyRecord] fiat_display_currency: str stake_currency: str @@ -304,7 +308,7 @@ class TradeSchema(BaseModel): min_rate: Optional[float] = None max_rate: Optional[float] = None - open_order_id: Optional[str] = None + has_open_orders: bool orders: List[OrderSchema] leverage: Optional[float] = None @@ -329,8 +333,6 @@ class OpenTradeSchema(TradeSchema): total_profit_fiat: Optional[float] = None total_profit_ratio: Optional[float] = None - open_order: Optional[str] = None - class TradeResponse(BaseModel): trades: List[TradeSchema] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7299364a8..8b1bb2a48 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -11,7 +11,7 @@ from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, - BlacklistResponse, Count, Daily, + BlacklistResponse, Count, DailyWeeklyMonthly, DeleteLockRequest, DeleteTrade, ExchangeListResponse, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, @@ -51,7 +51,8 @@ logger = logging.getLogger(__name__) # 2.30: new /pairlists endpoint # 2.31: new /backtest/history/ delete endpoint # 2.32: new /backtest/history/ patch endpoint -API_VERSION = 2.32 +# 2.33: Additional weekly/monthly metrics +API_VERSION = 2.33 # Public API, requires no auth. router_public = APIRouter() @@ -99,12 +100,24 @@ def stats(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stats() -@router.get('/daily', response_model=Daily, tags=['info']) +@router.get('/daily', response_model=DailyWeeklyMonthly, tags=['info']) def daily(timescale: int = 7, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return rpc._rpc_timeunit_profit(timescale, config['stake_currency'], config.get('fiat_display_currency', '')) +@router.get('/weekly', response_model=DailyWeeklyMonthly, tags=['info']) +def weekly(timescale: int = 4, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + return rpc._rpc_timeunit_profit(timescale, config['stake_currency'], + config.get('fiat_display_currency', ''), 'weeks') + + +@router.get('/monthly', response_model=DailyWeeklyMonthly, tags=['info']) +def monthly(timescale: int = 3, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + return rpc._rpc_timeunit_profit(timescale, config['stake_currency'], + config.get('fiat_display_currency', ''), 'months') + + @router.get('/status', response_model=List[OpenTradeSchema], tags=['info']) def status(rpc: RPC = Depends(get_rpc)): try: diff --git a/freqtrade/rpc/api_server/ws_schemas.py b/freqtrade/rpc/api_server/ws_schemas.py index 34eaf0245..970ea8cf8 100644 --- a/freqtrade/rpc/api_server/ws_schemas.py +++ b/freqtrade/rpc/api_server/ws_schemas.py @@ -5,7 +5,7 @@ from pandas import DataFrame from pydantic import BaseModel, ConfigDict from freqtrade.constants import PairWithTimeframe -from freqtrade.enums.rpcmessagetype import RPCMessageType, RPCRequestType +from freqtrade.enums import RPCMessageType, RPCRequestType class BaseArbitraryModel(BaseModel): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 61f4384b7..0abac3975 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,7 @@ from sqlalchemy import func, select from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange -from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config +from freqtrade.constants import CANCEL_REASON, Config from freqtrade.data.history import load_data from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection, @@ -26,12 +26,12 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange.types import Tickers from freqtrade.loggers import bufferHandler from freqtrade.misc import decimals_per_coin -from freqtrade.persistence import KeyStoreKeys, KeyValueStore, Order, PairLocks, Trade +from freqtrade.persistence import KeyStoreKeys, KeyValueStore, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc_types import RPCSendMsg -from freqtrade.util import dt_humanize, dt_now, shorten_date +from freqtrade.util import dt_humanize, dt_now, dt_ts_def, format_date, shorten_date from freqtrade.wallets import PositionWallet, Wallet @@ -171,11 +171,20 @@ class RPC: else: results = [] for trade in trades: - order: Optional[Order] = None current_profit_fiat: Optional[float] = None total_profit_fiat: Optional[float] = None - if trade.open_order_id: - order = trade.select_order_by_order_id(trade.open_order_id) + + # prepare open orders details + oo_details: Optional[str] = "" + oo_details_lst = [ + f'({oo.order_type} {oo.side} rem={oo.safe_remaining:.8f})' + for oo in trade.open_orders + if oo.ft_order_side not in ['stoploss'] + ] + oo_details = ', '.join(oo_details_lst) + + total_profit_abs = 0.0 + total_profit_ratio: Optional[float] = None # calculate profit and send message to user if trade.is_open: try: @@ -184,23 +193,22 @@ class RPC: except (ExchangeError, PricingError): current_rate = NAN if len(trade.select_filled_orders(trade.entry_side)) > 0: - current_profit = trade.calc_profit_ratio( - current_rate) if not isnan(current_rate) else NAN - current_profit_abs = trade.calc_profit( - current_rate) if not isnan(current_rate) else NAN + + current_profit = current_profit_abs = current_profit_fiat = NAN + if not isnan(current_rate): + prof = trade.calculate_profit(current_rate) + current_profit = prof.profit_ratio + current_profit_abs = prof.profit_abs + total_profit_abs = prof.total_profit + total_profit_ratio = prof.total_profit_ratio else: current_profit = current_profit_abs = current_profit_fiat = 0.0 + else: # Closed trade ... current_rate = trade.close_rate current_profit = trade.close_profit or 0.0 current_profit_abs = trade.close_profit_abs or 0.0 - total_profit_abs = trade.realized_profit + current_profit_abs - total_profit_ratio: Optional[float] = None - if trade.max_stake_amount: - total_profit_ratio = ( - (total_profit_abs / trade.max_stake_amount) * trade.leverage - ) # Calculate fiat profit if not isnan(current_profit_abs) and self._fiat_converter: @@ -216,8 +224,11 @@ class RPC: ) # Calculate guaranteed profit (in case of trailing stop) - stoploss_entry_dist = trade.calc_profit(trade.stop_loss) - stoploss_entry_dist_ratio = trade.calc_profit_ratio(trade.stop_loss) + stop_entry = trade.calculate_profit(trade.stop_loss) + + stoploss_entry_dist = stop_entry.profit_abs + stoploss_entry_dist_ratio = stop_entry.profit_ratio + # calculate distance to stoploss stoploss_current_dist = trade.stop_loss - current_rate stoploss_current_dist_ratio = stoploss_current_dist / current_rate @@ -230,7 +241,6 @@ class RPC: profit_pct=round(current_profit * 100, 2), profit_abs=current_profit_abs, profit_fiat=current_profit_fiat, - total_profit_abs=total_profit_abs, total_profit_fiat=total_profit_fiat, total_profit_ratio=total_profit_ratio, @@ -239,10 +249,7 @@ class RPC: stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2), stoploss_entry_dist=stoploss_entry_dist, stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8), - open_order=( - f'({order.order_type} {order.side} rem={order.safe_remaining:.8f})' if - order else None - ), + open_orders=oo_details )) results.append(trade_dict) return results @@ -267,8 +274,9 @@ class RPC: profit_str = f'{NAN:.2%}' else: if trade.nr_of_successful_entries > 0: - trade_profit = trade.calc_profit(current_rate) - profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}' + profit = trade.calculate_profit(current_rate) + trade_profit = profit.profit_abs + profit_str = f'{profit.profit_ratio:.2%}' else: trade_profit = 0.0 profit_str = f'{0.0:.2f}' @@ -283,18 +291,22 @@ class RPC: profit_str += f" ({fiat_profit:.2f})" fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \ else fiat_profit_sum + fiat_profit - open_order = (trade.select_order_by_order_id( - trade.open_order_id) if trade.open_order_id else None) + + active_attempt_side_symbols = [ + '*' if (oo and oo.ft_order_side == trade.entry_side) else '**' + for oo in trade.open_orders + ] + + # exemple: '*.**.**' trying to enter, exit and exit with 3 different orders + active_attempt_side_symbols_str = '.'.join(active_attempt_side_symbols) detail_trade = [ f'{trade.id} {direction_str}', - trade.pair + ('*' if (open_order - and open_order.ft_order_side == trade.entry_side) else '') - + ('**' if (open_order and - open_order.ft_order_side == trade.exit_side is not None) else ''), + trade.pair + active_attempt_side_symbols_str, shorten_date(dt_humanize(trade.open_date, only_distance=True)), profit_str ] + if self._config.get('position_adjustment_enable', False): max_entry_str = '' if self._config.get('max_entry_position_adjustment', -1) > 0: @@ -364,7 +376,7 @@ class RPC: data = [ { - 'date': f"{key.year}-{key.month:02d}" if timeunit == 'months' else key, + 'date': key, 'abs_profit': value["amount"], 'starting_balance': value["daily_stake"], 'rel_profit': value["rel_profit"], @@ -487,9 +499,10 @@ class RPC: profit_ratio = NAN profit_abs = NAN else: - profit_ratio = trade.calc_profit_ratio(rate=current_rate) - profit_abs = trade.calc_profit( - rate=trade.close_rate or current_rate) + trade.realized_profit + profit = trade.calculate_profit(trade.close_rate or current_rate) + + profit_ratio = profit.profit_ratio + profit_abs = profit.total_profit profit_all_coin.append(profit_abs) profit_all_ratio.append(profit_ratio) @@ -525,7 +538,8 @@ class RPC: winrate = (winning_trades / closed_trade_count) if closed_trade_count > 0 else 0 - trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), + trades_df = DataFrame([{'close_date': format_date(trade.close_date), + 'close_date_dt': trade.close_date, 'profit_abs': trade.close_profit_abs} for trade in trades if not trade.is_open and trade.close_date]) @@ -533,10 +547,15 @@ class RPC: max_drawdown_abs = 0.0 max_drawdown = 0.0 + drawdown_start: Optional[datetime] = None + drawdown_end: Optional[datetime] = None + dd_high_val = dd_low_val = 0.0 if len(trades_df) > 0: try: - (max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown( - trades_df, value_col='profit_abs', starting_balance=starting_balance) + (max_drawdown_abs, drawdown_start, drawdown_end, dd_high_val, dd_low_val, + max_drawdown) = calculate_max_drawdown( + trades_df, value_col='profit_abs', date_col='close_date_dt', + starting_balance=starting_balance) except ValueError: # ValueError if no losing trade. pass @@ -570,12 +589,12 @@ class RPC: 'profit_all_fiat': profit_all_fiat, 'trade_count': len(trades), 'closed_trade_count': closed_trade_count, - 'first_trade_date': first_date.strftime(DATETIME_PRINT_FORMAT) if first_date else '', + 'first_trade_date': format_date(first_date), 'first_trade_humanized': dt_humanize(first_date) if first_date else '', - 'first_trade_timestamp': int(first_date.timestamp() * 1000) if first_date else 0, - 'latest_trade_date': last_date.strftime(DATETIME_PRINT_FORMAT) if last_date else '', + 'first_trade_timestamp': dt_ts_def(first_date, 0), + 'latest_trade_date': format_date(last_date), 'latest_trade_humanized': dt_humanize(last_date) if last_date else '', - 'latest_trade_timestamp': int(last_date.timestamp() * 1000) if last_date else 0, + 'latest_trade_timestamp': dt_ts_def(last_date, 0), 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], 'best_pair': best_pair[0] if best_pair else '', 'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0, # Deprecated @@ -588,9 +607,15 @@ class RPC: 'expectancy_ratio': expectancy_ratio, 'max_drawdown': max_drawdown, 'max_drawdown_abs': max_drawdown_abs, + 'max_drawdown_start': format_date(drawdown_start), + 'max_drawdown_start_timestamp': dt_ts_def(drawdown_start), + 'max_drawdown_end': format_date(drawdown_end), + 'max_drawdown_end_timestamp': dt_ts_def(drawdown_end), + 'drawdown_high': dd_high_val, + 'drawdown_low': dd_low_val, 'trading_volume': trading_volume, - 'bot_start_timestamp': int(bot_start.timestamp() * 1000) if bot_start else 0, - 'bot_start_date': bot_start.strftime(DATETIME_PRINT_FORMAT) if bot_start else '', + 'bot_start_timestamp': dt_ts_def(bot_start, 0), + 'bot_start_date': format_date(bot_start), } def __balance_get_est_stake( @@ -762,21 +787,25 @@ class RPC: def __exec_force_exit(self, trade: Trade, ordertype: Optional[str], amount: Optional[float] = None) -> bool: - # Check if there is there is an open order - fully_canceled = False - if trade.open_order_id: - order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) + # Check if there is there are open orders + trade_entry_cancelation_registry = [] + for oo in trade.open_orders: + trade_entry_cancelation_res = {'order_id': oo.order_id, 'cancel_state': False} + order = self._freqtrade.exchange.fetch_order(oo.order_id, trade.pair) if order['side'] == trade.entry_side: fully_canceled = self._freqtrade.handle_cancel_enter( - trade, order, CANCEL_REASON['FORCE_EXIT']) + trade, order, oo.order_id, CANCEL_REASON['FORCE_EXIT']) + trade_entry_cancelation_res['cancel_state'] = fully_canceled + trade_entry_cancelation_registry.append(trade_entry_cancelation_res) if order['side'] == trade.exit_side: # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT']) + self._freqtrade.handle_cancel_exit( + trade, order, oo.order_id, CANCEL_REASON['FORCE_EXIT']) - if not fully_canceled: - if trade.open_order_id is not None: + if all(tocr['cancel_state'] is False for tocr in trade_entry_cancelation_registry): + if trade.has_open_orders: # Order cancellation failed, so we can't exit. return False # Get current rate and execute sell @@ -875,10 +904,10 @@ class RPC: if trade: is_short = trade.is_short if not self._freqtrade.strategy.position_adjustment_enable: - raise RPCException(f'position for {pair} already open - id: {trade.id}') - if trade.open_order_id is not None: - raise RPCException(f'position for {pair} already open - id: {trade.id} ' - f'and has open order {trade.open_order_id}') + raise RPCException(f"position for {pair} already open - id: {trade.id}") + if trade.has_open_orders: + raise RPCException(f"position for {pair} already open - id: {trade.id} " + f"and has open order {','.join(trade.open_orders_ids)}") else: if Trade.get_open_trade_count() >= self._config['max_open_trades']: raise RPCException("Maximum number of trades is reached.") @@ -915,16 +944,18 @@ class RPC: if not trade: logger.warning('cancel_open_order: Invalid trade_id received.') raise RPCException('Invalid trade_id.') - if not trade.open_order_id: + if not trade.has_open_orders: logger.warning('cancel_open_order: No open order for trade_id.') raise RPCException('No open order for trade_id.') - try: - order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) - except ExchangeError as e: - logger.info(f"Cannot query order for {trade} due to {e}.", exc_info=True) - raise RPCException("Order not found.") - self._freqtrade.handle_cancel_order(order, trade, CANCEL_REASON['USER_CANCEL']) + for open_order in trade.open_orders: + try: + order = self._freqtrade.exchange.fetch_order(open_order.order_id, trade.pair) + except ExchangeError as e: + logger.info(f"Cannot query order for {trade} due to {e}.", exc_info=True) + raise RPCException("Order not found.") + self._freqtrade.handle_cancel_order( + order, open_order.order_id, trade, CANCEL_REASON['USER_CANCEL']) Trade.commit() def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]: @@ -940,9 +971,9 @@ class RPC: raise RPCException('invalid argument') # Try cancelling regular order if that exists - if trade.open_order_id: + for open_order in trade.open_orders: try: - self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) + self._freqtrade.exchange.cancel_order(open_order.order_id, trade.pair) c_count += 1 except (ExchangeError): pass @@ -1092,7 +1123,7 @@ class RPC: buffer = bufferHandler.buffer[-limit:] else: buffer = bufferHandler.buffer - records = [[datetime.fromtimestamp(r.created).strftime(DATETIME_PRINT_FORMAT), + records = [[format_date(datetime.fromtimestamp(r.created)), r.created * 1000, r.name, r.levelname, r.message + ('\n' + r.exc_text if r.exc_text else '')] for r in buffer] @@ -1309,7 +1340,7 @@ class RPC: return { "last_process": str(last_p), - "last_process_loc": last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT), + "last_process_loc": format_date(last_p.astimezone(tzlocal())), "last_process_ts": int(last_p.timestamp()), } diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index aced89d7a..be27c38f4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -51,6 +51,7 @@ class TimeunitMappings: message2: str callback: str default: int + dateformat: str def authorized_only(command_handler: Callable[..., Coroutine[Any, Any, None]]): @@ -531,40 +532,24 @@ class Telegram(RPCHandler): cur_entry_amount = order["filled"] or order["amount"] cur_entry_average = order["safe_price"] lines.append(" ") + lines.append(f"*{wording} #{order_nr}:*") if order_nr == 1: - lines.append(f"*{wording} #{order_nr}:*") lines.append( f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})" ) lines.append(f"*Average Price:* {cur_entry_average:.8g}") else: - sum_stake = 0 - sum_amount = 0 - for y in range(order_nr): - loc_order = filled_orders[y] - if loc_order['is_open'] is True: - # Skip open orders (e.g. stop orders) - continue - amount = loc_order["filled"] or loc_order["amount"] - sum_stake += amount * loc_order["safe_price"] - sum_amount += amount - prev_avg_price = sum_stake / sum_amount # TODO: This calculation ignores fees. price_to_1st_entry = ((cur_entry_average - first_avg) / first_avg) - minus_on_entry = 0 - if prev_avg_price: - minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price - - lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg Profit") if is_open: lines.append("({})".format(dt_humanize(order["order_filled_date"], granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} " - f"({price_to_1st_entry:.2%} from 1st entry Rate)") - lines.append(f"*Order filled:* {order['order_filled_date']}") + f"({price_to_1st_entry:.2%} from 1st entry rate)") + lines.append(f"*Order Filled:* {order['order_filled_date']}") lines_detail.append("\n".join(lines)) @@ -662,10 +647,10 @@ class Telegram(RPCHandler): ("`({stop_loss_ratio:.2%})`" if r['stop_loss_ratio'] else "")) lines.append("*Stoploss distance:* `{stoploss_current_dist:.8g}` " "`({stoploss_current_dist_ratio:.2%})`") - if r['open_order']: + if r.get('open_orders'): lines.append( - "*Open Order:* `{open_order}`" - + "- `{exit_order_status}`" if r['exit_order_status'] else "") + "*Open Order:* `{open_orders}`" + + ("- `{exit_order_status}`" if r['exit_order_status'] else "")) lines_detail = self._prepare_order_details( r['orders'], r['quote_currency'], r['is_open']) @@ -736,10 +721,10 @@ class Telegram(RPCHandler): """ vals = { - 'days': TimeunitMappings('Day', 'Daily', 'days', 'update_daily', 7), + 'days': TimeunitMappings('Day', 'Daily', 'days', 'update_daily', 7, '%Y-%m-%d'), 'weeks': TimeunitMappings('Monday', 'Weekly', 'weeks (starting from Monday)', - 'update_weekly', 8), - 'months': TimeunitMappings('Month', 'Monthly', 'months', 'update_monthly', 6), + 'update_weekly', 8, '%Y-%m-%d'), + 'months': TimeunitMappings('Month', 'Monthly', 'months', 'update_monthly', 6, '%Y-%m'), } val = vals[unit] @@ -756,7 +741,7 @@ class Telegram(RPCHandler): unit ) stats_tab = tabulate( - [[f"{period['date']} ({period['trade_count']})", + [[f"{period['date']:{val.dateformat}} ({period['trade_count']})", f"{round_coin_value(period['abs_profit'], stats['stake_currency'])}", f"{period['fiat_value']:.2f} {stats['fiat_display_currency']}", f"{period['rel_profit']:.2%}", @@ -888,7 +873,11 @@ class Telegram(RPCHandler): f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n" f"*Profit factor:* `{stats['profit_factor']:.2f}`\n" f"*Max Drawdown:* `{stats['max_drawdown']:.2%} " - f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`" + f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`\n" + f" from `{stats['max_drawdown_start']} " + f"({round_coin_value(stats['drawdown_high'], stake_cur)})`\n" + f" to `{stats['max_drawdown_end']} " + f"({round_coin_value(stats['drawdown_low'], stake_cur)})`\n" ) await self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit", query=update.callback_query) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index eca3e3ede..5cdbb6bf6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -373,7 +373,7 @@ class IStrategy(ABC, HyperStrategyMixin): return True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + current_profit: float, after_fill: bool, **kwargs) -> Optional[float]: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -389,12 +389,14 @@ class IStrategy(ABC, HyperStrategyMixin): :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param after_fill: True if the stoploss is called after the order was filled. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New stoploss value, relative to the current_rate """ return self.stoploss - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], + current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -404,6 +406,7 @@ class IStrategy(ABC, HyperStrategyMixin): When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -512,7 +515,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -521,8 +524,9 @@ class IStrategy(ABC, HyperStrategyMixin): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. @@ -719,6 +723,8 @@ class IStrategy(ABC, HyperStrategyMixin): # END - Intended to be overridden by strategy ### + _ft_stop_uses_after_fill = False + def __informative_pairs_freqai(self) -> ListPairsWithTimeframes: """ Create informative-pairs needed for FreqAI @@ -1160,13 +1166,17 @@ class IStrategy(ABC, HyperStrategyMixin): def ft_stoploss_adjust(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, force_stoploss: float, low: Optional[float] = None, - high: Optional[float] = None) -> None: + high: Optional[float] = None, after_fill: bool = False) -> None: """ Adjust stop-loss dynamically if configured to do so. :param current_profit: current profit as ratio :param low: Low value of this candle, only set in backtesting :param high: High value of this candle, only set in backtesting """ + if after_fill and not self._ft_stop_uses_after_fill: + # Skip if the strategy doesn't support after fill. + return + stop_loss_value = force_stoploss if force_stoploss else self.stoploss # Initiate stoploss with open_rate. Does nothing if stoploss is already set. @@ -1181,18 +1191,20 @@ class IStrategy(ABC, HyperStrategyMixin): bound = (low if trade.is_short else high) bound_profit = current_profit if not bound else trade.calc_profit_ratio(bound) if self.use_custom_stoploss and dir_correct: - stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None, - supress_error=True - )(pair=trade.pair, trade=trade, - current_time=current_time, - current_rate=(bound or current_rate), - current_profit=bound_profit) + stop_loss_value_custom = strategy_safe_wrapper( + self.custom_stoploss, default_retval=None, supress_error=True + )(pair=trade.pair, trade=trade, + current_time=current_time, + current_rate=(bound or current_rate), + current_profit=bound_profit, + after_fill=after_fill) # Sanity check - error cases will return None - if stop_loss_value: - # logger.info(f"{trade.pair} {stop_loss_value=} {bound_profit=}") - trade.adjust_stop_loss(bound or current_rate, stop_loss_value) + if stop_loss_value_custom: + stop_loss_value = stop_loss_value_custom + trade.adjust_stop_loss(bound or current_rate, stop_loss_value, + allow_refresh=after_fill) else: - logger.warning("CustomStoploss function did not return valid stoploss") + logger.debug("CustomStoploss function did not return valid stoploss") if self.trailing_stop and dir_correct: # trailing stoploss handling @@ -1245,7 +1257,7 @@ class IStrategy(ABC, HyperStrategyMixin): exit_type = ExitType.STOP_LOSS # If initial stoploss is not the same as current one then it is trailing. - if trade.initial_stop_loss != trade.stop_loss: + if trade.is_stop_loss_trailing: exit_type = ExitType.TRAILING_STOP_LOSS logger.debug( f"{trade.pair} - HIT STOP: current price at " diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 27ebe7e69..7654a383f 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -45,10 +45,13 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, elif minutes < minutes_inf: # Subtract "small" timeframe so merging is not delayed by 1 small candle # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 - informative['date_merge'] = ( - informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - - pd.to_timedelta(minutes, 'm') - ) + if not informative.empty: + informative['date_merge'] = ( + informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - + pd.to_timedelta(minutes, 'm') + ) + else: + informative['date_merge'] = informative[date_column] else: raise ValueError("Tried to merge a faster timeframe to a slower timeframe." "This would create new rows, and can throw off your regular indicators.") @@ -123,7 +126,8 @@ def stoploss_from_open( return max(stoploss * leverage, 0.0) -def stoploss_from_absolute(stop_rate: float, current_rate: float, is_short: bool = False) -> float: +def stoploss_from_absolute(stop_rate: float, current_rate: float, is_short: bool = False, + leverage: float = 1.0) -> float: """ Given current price and desired stop price, return a stop loss value that is relative to current price. @@ -136,6 +140,7 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float, is_short: bool :param stop_rate: Stop loss price. :param current_rate: Current asset price. :param is_short: When true, perform the calculation for short instead of long + :param leverage: Leverage to use for the calculation :return: Positive stop loss value relative to current price """ @@ -150,4 +155,4 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float, is_short: bool # negative stoploss values indicate the requested stop price is higher/lower # (long/short) than the current price # shorts can yield stoploss values higher than 1, so limit that as well - return max(min(stoploss, 1.0), 0.0) + return max(min(stoploss, 1.0), 0.0) * leverage diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 084cf2e89..e64570b9e 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy): plot_config = { "main_plot": {}, "subplots": { - "&-s_close": {"prediction": {"color": "blue"}}, + "&-s_close": {"&-s_close": {"color": "blue"}}, "do_predict": { "do_predict": {"color": "brown"}, }, diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index fd81570fe..dec547715 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -77,7 +77,7 @@ class SampleStrategy(IStrategy): exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 30 + startup_candle_count: int = 200 # Optional order type mapping. order_types = { diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 2bfa4155d..0b30dbd54 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -243,7 +243,7 @@ "# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)\n", "\n", "from freqtrade.configuration import Configuration\n", - "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n", + "from freqtrade.data.btanalysis import load_backtest_stats\n", "import plotly.express as px\n", "import pandas as pd\n", "\n", @@ -254,20 +254,8 @@ "stats = load_backtest_stats(backtest_dir)\n", "strategy_stats = stats['strategy'][strategy]\n", "\n", - "dates = []\n", - "profits = []\n", - "for date_profit in strategy_stats['daily_profit']:\n", - " dates.append(date_profit[0])\n", - " profits.append(date_profit[1])\n", - "\n", - "equity = 0\n", - "equity_daily = []\n", - "for daily_profit in profits:\n", - " equity_daily.append(equity)\n", - " equity += float(daily_profit)\n", - "\n", - "\n", - "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n", + "df = pd.DataFrame(columns=['dates','equity'], data=strategy_stats['daily_profit'])\n", + "df['equity_daily'] = df['equity'].cumsum()\n", "\n", "fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n", "fig.show()\n" @@ -414,7 +402,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.4" }, "mimetype": "text/x-python", "name": "python", diff --git a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 index bfbb20ec1..6fad129c7 100644 --- a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 @@ -13,7 +13,8 @@ def bot_loop_start(self, current_time: datetime, **kwargs) -> None: """ pass -def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: float, +def custom_entry_price(self, pair: str, trade: Optional['Trade'], + current_time: 'datetime', proposed_rate: float, entry_tag: 'Optional[str]', side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -23,6 +24,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -102,8 +104,8 @@ def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: f use_custom_stoploss = True -def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', - current_rate: float, current_profit: float, **kwargs) -> float: +def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, + current_profit: float, after_fill: bool, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -111,7 +113,7 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ - When not implemented by a strategy, returns the initial stoploss value + When not implemented by a strategy, returns the initial stoploss value. Only called when use_custom_stoploss is set to True. :param pair: Pair that's currently analyzed @@ -119,10 +121,10 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in exit_pricing. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param after_fill: True if the stoploss is called after the order was filled. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New stoploss value, relative to the current_rate """ - return self.stoploss def custom_exit(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, current_profit: float, **kwargs) -> 'Optional[Union[str, bool]]': @@ -257,7 +259,7 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -266,8 +268,9 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. @@ -276,8 +279,8 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade, - Positive values to increase position, Negative values to decrease position. - Return None for no action. + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ return None diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 index 023862314..118e8104e 100644 --- a/freqtrade/templates/subtemplates/exchange_bittrex.j2 +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -9,11 +9,8 @@ "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 500 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 index 4d0e4c1ff..7dd646720 100644 --- a/freqtrade/templates/subtemplates/exchange_kraken.j2 +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -3,13 +3,8 @@ "name": "kraken", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 1000 - // Enable the below for downoading data. - //"rateLimit": 3100 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index af09624ac..fc7cf5f6a 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,5 +1,6 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, - dt_utc, format_ms_time, shorten_date) + dt_ts_def, dt_utc, format_date, format_ms_time, + shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache from freqtrade.util.template_renderer import render_template, render_template_with_fallback # noqa @@ -11,7 +12,9 @@ __all__ = [ 'dt_humanize', 'dt_now', 'dt_ts', + 'dt_ts_def', 'dt_utc', + 'format_date', 'format_ms_time', 'FtPrecise', 'PeriodicCache', diff --git a/freqtrade/util/binance_mig.py b/freqtrade/util/binance_mig.py index 7043459f2..cc9c451b2 100644 --- a/freqtrade/util/binance_mig.py +++ b/freqtrade/util/binance_mig.py @@ -4,7 +4,7 @@ from packaging import version from sqlalchemy import select from freqtrade.constants import DOCS_LINK, Config -from freqtrade.enums.tradingmode import TradingMode +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Trade diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 7f44cbdb0..102c83143 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -4,6 +4,8 @@ from typing import Optional import arrow +from freqtrade.constants import DATETIME_PRINT_FORMAT + def dt_now() -> datetime: """Return the current datetime in UTC.""" @@ -26,6 +28,16 @@ def dt_ts(dt: Optional[datetime] = None) -> int: return int(dt_now().timestamp() * 1000) +def dt_ts_def(dt: Optional[datetime], default: int = 0) -> int: + """ + Return dt in ms as a timestamp in UTC. + If dt is None, return the current datetime in UTC. + """ + if dt: + return int(dt.timestamp() * 1000) + return default + + def dt_floor_day(dt: datetime) -> datetime: """Return the floor of the day for the given datetime.""" return dt.replace(hour=0, minute=0, second=0, microsecond=0) @@ -63,6 +75,17 @@ def dt_humanize(dt: datetime, **kwargs) -> str: return arrow.get(dt).humanize(**kwargs) +def format_date(date: Optional[datetime]) -> str: + """ + Return a formatted date string. + Returns an empty string if date is None. + :param date: datetime to format + """ + if date: + return date.strftime(DATETIME_PRINT_FORMAT) + return '' + + def format_ms_time(date: int) -> str: """ convert MS date to readable format. diff --git a/mkdocs.yml b/mkdocs.yml index bb5ae0010..53894d007 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,6 @@ nav: - Web Hook: webhook-config.md - Data Downloading: data-download.md - Backtesting: backtesting.md - - Lookahead analysis: lookahead-analysis.md - Hyperopt: hyperopt.md - FreqAI: - Introduction: freqai.md @@ -43,6 +42,8 @@ nav: - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Trade Object: trade-object.md + - Lookahead analysis: lookahead-analysis.md + - Recursive analysis: recursive-analysis.md - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md - Producer/Consumer mode: producer-consumer.md diff --git a/pyproject.toml b/pyproject.toml index 40c0e2005..cd0c65916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ ignore = ["freqtrade/vendor/**"] line-length = 100 extend-exclude = [".env", ".venv"] target-version = "py38" +# Exclude UP036 as it's causing the "exit if < 3.9" to fail. extend-select = [ "C90", # mccabe # "N", # pep8-naming diff --git a/requirements-dev.txt b/requirements-dev.txt index cac9d5cd7..ba01a71c2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,24 +7,24 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.285 +ruff==0.0.291 mypy==1.5.1 -pre-commit==3.3.3 -pytest==7.4.0 +pre-commit==3.4.0 +pytest==7.4.2 pytest-asyncio==0.21.1 pytest-cov==4.1.0 pytest-mock==3.11.1 pytest-random-order==1.1.0 isort==5.12.0 # For datetime mocking -time-machine==2.12.0 +time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.7.4 +nbconvert==7.8.0 # mypy types types-cachetools==5.3.0.6 types-filelock==3.2.7 -types-requests==2.31.0.2 +types-requests==2.31.0.4 types-tabulate==0.9.0.3 types-python-dateutil==2.8.19.14 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index d8421e968..0cd68df2a 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,8 +5,8 @@ # Required for freqai scikit-learn==1.1.3 joblib==1.3.2 -catboost==1.2; 'arm' not in platform_machine -lightgbm==4.0.0 -xgboost==1.7.6 +catboost==1.2.2; 'arm' not in platform_machine +lightgbm==4.1.0 +xgboost==2.0.0 tensorboard==2.14.0 datasieve==0.1.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 80b62e2f3..06f8ddbaf 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,8 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.11.2; python_version >= '3.9' -scipy==1.10.1; python_version < '3.9' +scipy==1.11.2 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.12.2 +filelock==3.12.4 diff --git a/requirements-plot.txt b/requirements-plot.txt index 9a8c596ad..b2ec35539 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.16.1 +plotly==5.17.0 diff --git a/requirements.txt b/requirements.txt index bf346382f..efcb478ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,20 @@ -numpy==1.25.2; python_version > '3.8' -numpy==1.24.3; python_version <= '3.8' +numpy==1.26.0; platform_machine != 'armv7l' +numpy==1.25.2; platform_machine == 'armv7l' pandas==2.0.3 pandas-ta==0.3.14b -ccxt==4.0.71 -cryptography==41.0.3; platform_machine != 'armv7l' -cryptography==40.0.1; platform_machine == 'armv7l' +ccxt==4.0.105 +cryptography==41.0.3 aiohttp==3.8.5 -SQLAlchemy==2.0.20 -python-telegram-bot==20.4 +SQLAlchemy==2.0.21 +python-telegram-bot==20.5 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 arrow==1.2.3 cachetools==5.3.1 requests==2.31.0 -urllib3==2.0.4 -jsonschema==4.19.0 +urllib3==2.0.5 +jsonschema==4.19.1 TA-Lib==0.4.28 technical==1.4.0 tabulate==0.9.0 @@ -24,23 +23,23 @@ jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 joblib==1.3.2 -rich==13.5.2 -pyarrow==12.0.1; platform_machine != 'armv7l' +rich==13.5.3 +pyarrow==13.0.0; platform_machine != 'armv7l' # find first, C search in arrays py_find_1st==1.1.5 # Load ticker files 30% faster -python-rapidjson==1.10 +python-rapidjson==1.11 # Properly format api responses -orjson==3.9.5 +orjson==3.9.7 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.101.1 -pydantic==2.2.1 +fastapi==0.103.1 +pydantic==2.3.0 uvicorn==0.23.2 pyjwt==2.8.0 aiofiles==23.2.1 @@ -49,7 +48,7 @@ psutil==5.9.5 # Support for colorized terminal output colorama==0.4.6 # Building config files interactively -questionary==2.0.0 +questionary==2.0.1 prompt-toolkit==3.0.36 # Extensions to datetime library python-dateutil==2.8.2 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 2b4690287..dfe50cc2c 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -134,6 +134,20 @@ class FtRestClient: """ return self._get("daily", params={"timescale": days} if days else None) + def weekly(self, weeks=None): + """Return the profits for each week, and amount of trades. + + :return: json object + """ + return self._get("weekly", params={"timescale": weeks} if weeks else None) + + def monthly(self, months=None): + """Return the profits for each month, and amount of trades. + + :return: json object + """ + return self._get("monthly", params={"timescale": months} if months else None) + def edge(self): """Return information about edge. diff --git a/setup.cfg b/setup.cfg index b54b62619..d4d70bc34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ classifiers = Environment :: Console Intended Audience :: Science/Research License :: OSI Approved :: GNU General Public License v3 (GPLv3) - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -33,7 +32,7 @@ tests_require = pytest-mock packages = find: -python_requires = >=3.8 +python_requires = >=3.9 [options.entry_points] console_scripts = @@ -50,3 +49,5 @@ exclude = __pycache__, .eggs, user_data, + .venv + .env diff --git a/setup.sh b/setup.sh index cd9b85d2a..6bf85edab 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ function check_installed_python() { exit 2 fi - for v in 11 10 9 8 + for v in 11 10 9 do PYTHON="python3.${v}" which $PYTHON @@ -36,7 +36,7 @@ function check_installed_python() { fi done - echo "No usable python found. Please make sure to have python3.8 or newer installed." + echo "No usable python found. Please make sure to have python3.9 or newer installed." exit 1 } @@ -192,7 +192,7 @@ function update() { fi updateenv echo "Update completed." - echo_block "Don't forget to activate your virtual enviorment with 'source .venv/bin/activate'!" + echo_block "Don't forget to activate your virtual environment with 'source .venv/bin/activate'!" } @@ -277,7 +277,7 @@ function install() { install_redhat else echo "This script does not support your OS." - echo "If you have Python version 3.8 - 3.11, pip, virtualenv, ta-lib you can continue." + echo "If you have Python version 3.9 - 3.11, pip, virtualenv, ta-lib you can continue." echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi @@ -304,7 +304,7 @@ function help() { echo " -p,--plot Install dependencies for Plotting scripts." } -# Verify if 3.8+ is installed +# Verify if 3.9+ is installed check_installed_python case $* in diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cdaaf9654..e066eac01 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1556,7 +1556,7 @@ def test_start_strategy_updater(mocker, tmpdir): pargs['config'] = None start_strategy_update(pargs) # Number of strategies in the test directory - assert sc_mock.call_count == 11 + assert sc_mock.call_count == 12 sc_mock.reset_mock() args = [ diff --git a/tests/conftest.py b/tests/conftest.py index 732fffd8f..4372534ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2601,7 +2601,6 @@ def open_trade(): pair='ETH/BTC', open_rate=0.00001099, exchange='binance', - open_order_id='123456789', amount=90.99181073, fee_open=0.0, fee_close=0.0, @@ -2613,7 +2612,7 @@ def open_trade(): Order( ft_order_side='buy', ft_pair=trade.pair, - ft_is_open=False, + ft_is_open=True, ft_amount=trade.amount, ft_price=trade.open_rate, order_id='123456789', @@ -2639,7 +2638,6 @@ def open_trade_usdt(): pair='ADA/USDT', open_rate=2.0, exchange='binance', - open_order_id='123456789_exit', amount=30.0, fee_open=0.0, fee_close=0.0, diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index e50ed0e59..a2276ae16 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -46,7 +46,6 @@ def mock_trade_1(fee, is_short: bool): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=0.123, exchange='binance', - open_order_id=f'dry_run_buy_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, is_short=is_short @@ -210,7 +209,6 @@ def mock_trade_4(fee, is_short: bool): is_open=True, open_rate=0.123, exchange='binance', - open_order_id=f'prod_buy_{direc(is_short)}_12345', strategy='StrategyTestV3', timeframe=5, is_short=is_short, @@ -327,7 +325,6 @@ def mock_trade_6(fee, is_short: bool): exchange='binance', strategy='SampleStrategy', enter_tag='TEST2', - open_order_id=f"prod_sell_{direc(is_short)}_6", timeframe=5, is_short=is_short ) @@ -411,7 +408,6 @@ def short_trade(fee): # close_profit_abs=-0.6925113200000013, exchange='binance', is_open=True, - open_order_id=None, strategy='DefaultStrategy', timeframe=5, exit_reason='sell_signal', @@ -502,7 +498,6 @@ def leverage_trade(fee): close_profit_abs=2.5983135000000175, exchange='kraken', is_open=False, - open_order_id='dry_run_leverage_buy_12368', strategy='DefaultStrategy', timeframe=5, exit_reason='sell_signal', diff --git a/tests/conftest_trades_usdt.py b/tests/conftest_trades_usdt.py index fc4776148..d73a53605 100644 --- a/tests/conftest_trades_usdt.py +++ b/tests/conftest_trades_usdt.py @@ -66,7 +66,6 @@ def mock_trade_usdt_1(fee, is_short: bool): close_profit_abs=-4.09, exchange='binance', strategy='SampleStrategy', - open_order_id=f'prod_exit_1_{direc(is_short)}', timeframe=5, is_short=is_short, ) @@ -123,7 +122,6 @@ def mock_trade_usdt_2(fee, is_short: bool): close_profit_abs=3.9875, exchange='binance', is_open=False, - open_order_id=f'12366_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, enter_tag='TEST1', @@ -231,7 +229,6 @@ def mock_trade_usdt_4(fee, is_short: bool): is_open=True, open_rate=2.0, exchange='binance', - open_order_id=f'prod_buy_12345_{direc(is_short)}', strategy='StrategyTestV2', timeframe=5, is_short=is_short, @@ -340,7 +337,6 @@ def mock_trade_usdt_6(fee, is_short: bool): open_rate=10.0, exchange='binance', strategy='SampleStrategy', - open_order_id=f'prod_exit_6_{direc(is_short)}', timeframe=5, is_short=is_short, ) @@ -378,7 +374,6 @@ def mock_trade_usdt_7(fee, is_short: bool): open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=2.0, exchange='binance', - open_order_id=None, strategy='StrategyTestV2', timeframe=5, is_short=is_short, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 7ca236442..74bb8d9f9 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -6,12 +6,14 @@ from shutil import copyfile import numpy as np import pandas as pd import pytest +from pandas.testing import assert_frame_equal from freqtrade.configuration.timerange import TimeRange from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, - ohlcv_fill_up_missing_data, ohlcv_to_dataframe, - reduce_dataframe_footprint, trades_df_remove_duplicates, - trades_dict_to_list, trades_to_ohlcv, trim_dataframe) + convert_trades_to_ohlcv, ohlcv_fill_up_missing_data, + ohlcv_to_dataframe, reduce_dataframe_footprint, + trades_df_remove_duplicates, trades_dict_to_list, + trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from freqtrade.data.history.idatahandler import IDataHandler @@ -441,3 +443,39 @@ def test_reduce_dataframe_footprint(): # Changes dtype of returned dataframe assert df2['open_copy'].dtype == np.float32 assert df2['close_copy'].dtype == np.float32 + + +def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): + tmpdir1 = Path(tmpdir) + pair = 'XRP/ETH' + file1 = tmpdir1 / 'XRP_ETH-1m.feather' + file5 = tmpdir1 / 'XRP_ETH-5m.feather' + filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file1.name, file1) + copyfile(testdatadir / file5.name, file5) + copyfile(testdatadir / filetrades.name, filetrades) + + # Compare downloaded dataset with converted dataset + dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) + + tr = TimeRange.parse_timerange('20191011-20191012') + + convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], + data_format_trades='jsongz', + datadir=tmpdir1, timerange=tr, erase=True) + + assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) + # Load new data + df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) + + assert_frame_equal(dfbak_1m, df_1m, check_exact=True) + assert_frame_equal(dfbak_5m, df_5m, check_exact=True) + msg = 'Could not convert NoDatapair to OHLCV.' + assert not log_has(msg, caplog) + + convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], + data_format_trades='jsongz', + datadir=tmpdir1, timerange=tr, erase=True) + assert log_has(msg, caplog) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 84f00b8f4..13fdb034b 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -16,9 +16,9 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.data.history.history_utils import (_download_pair_history, _download_trades_history, - _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timerange, load_data, - load_pair_history, refresh_backtest_ohlcv_data, + _load_cached_data_for_updating, get_timerange, + load_data, load_pair_history, + refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, validate_backtest_data) from freqtrade.data.history.idatahandler import get_datahandler @@ -632,39 +632,3 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert ght_mock.call_args_list[0][1]['from_id'] is None assert log_has_re(r'Start .* earlier than available data. Redownloading trades for.*', caplog) _clean_test_file(file2) - - -def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): - tmpdir1 = Path(tmpdir) - pair = 'XRP/ETH' - file1 = tmpdir1 / 'XRP_ETH-1m.feather' - file5 = tmpdir1 / 'XRP_ETH-5m.feather' - filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz' - copyfile(testdatadir / file1.name, file1) - copyfile(testdatadir / file5.name, file5) - copyfile(testdatadir / filetrades.name, filetrades) - - # Compare downloaded dataset with converted dataset - dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) - dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) - - tr = TimeRange.parse_timerange('20191011-20191012') - - convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], - data_format_trades='jsongz', - datadir=tmpdir1, timerange=tr, erase=True) - - assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) - # Load new data - df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) - df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) - - assert_frame_equal(dfbak_1m, df_1m, check_exact=True) - assert_frame_equal(dfbak_5m, df_5m, check_exact=True) - msg = 'Could not convert NoDatapair to OHLCV.' - assert not log_has(msg, caplog) - - convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], - data_format_trades='jsongz', - datadir=tmpdir1, timerange=tr, erase=True) - assert log_has(msg, caplog) diff --git a/tests/exchange/test_bybit.py b/tests/exchange/test_bybit.py index 7495f543b..f7383934b 100644 --- a/tests/exchange/test_bybit.py +++ b/tests/exchange/test_bybit.py @@ -1,9 +1,9 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock from freqtrade.enums.marginmode import MarginMode from freqtrade.enums.tradingmode import TradingMode -from tests.conftest import get_mock_coro, get_patched_exchange +from tests.conftest import EXMS, get_mock_coro, get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -68,3 +68,83 @@ def test_bybit_get_funding_fees(default_conf, mocker): exchange.get_funding_fees('BTC/USDT:USDT', 1, False, now) assert exchange._fetch_and_calculate_funding_fees.call_count == 1 + + +def test_bybit_fetch_orders(default_conf, mocker, limit_order): + + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[ + limit_order['buy'], + limit_order['sell'], + ]) + api_mock.fetch_open_orders = MagicMock(return_value=[limit_order['buy']]) + api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='bybit') + # Not available in dry-run + assert exchange.fetch_orders('mocked', start_time) == [] + assert api_mock.fetch_orders.call_count == 0 + default_conf['dry_run'] = False + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='bybit') + res = exchange.fetch_orders('mocked', start_time) + # Bybit will call the endpoint 3 times, as it has a limit of 7 days per call + assert api_mock.fetch_orders.call_count == 3 + assert api_mock.fetch_open_orders.call_count == 0 + assert api_mock.fetch_closed_orders.call_count == 0 + assert len(res) == 2 * 3 + + +def test_bybit_fetch_order_canceled_empty(default_conf_usdt, mocker): + default_conf_usdt['dry_run'] = False + + api_mock = MagicMock() + api_mock.fetch_order = MagicMock(return_value={ + 'id': '123', + 'symbol': 'BTC/USDT', + 'status': 'canceled', + 'filled': 0.0, + 'remaining': 0.0, + 'amount': 20.0, + }) + + exchange = get_patched_exchange(mocker, default_conf_usdt, api_mock, id='bybit') + + res = exchange.fetch_order('123', 'BTC/USDT') + assert res['remaining'] is None + assert res['filled'] == 0.0 + assert res['amount'] == 20.0 + assert res['status'] == 'canceled' + + api_mock.fetch_order = MagicMock(return_value={ + 'id': '123', + 'symbol': 'BTC/USDT', + 'status': 'canceled', + 'filled': 0.0, + 'remaining': 20.0, + 'amount': 20.0, + }) + # Don't touch orders which return correctly. + res1 = exchange.fetch_order('123', 'BTC/USDT') + assert res1['remaining'] == 20.0 + assert res1['filled'] == 0.0 + assert res1['amount'] == 20.0 + assert res1['status'] == 'canceled' + + # Reverse test - remaining is not touched + api_mock.fetch_order = MagicMock(return_value={ + 'id': '124', + 'symbol': 'BTC/USDT', + 'status': 'open', + 'filled': 0.0, + 'remaining': 20.0, + 'amount': 20.0, + }) + res2 = exchange.fetch_order('123', 'BTC/USDT') + assert res2['remaining'] == 20.0 + assert res2['filled'] == 0.0 + assert res2['amount'] == 20.0 + assert res2['status'] == 'open' diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index adb8b7497..7320f1074 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -7,20 +7,16 @@ from unittest.mock import MagicMock, Mock, PropertyMock, patch import ccxt import pytest -from ccxt import DECIMAL_PLACES, ROUND, ROUND_UP, TICK_SIZE, TRUNCATE from pandas import DataFrame from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) -from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_precision, - date_minus_candles, market_is_active, price_to_precision, - timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, - timeframe_to_prev_date, timeframe_to_seconds) +from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, market_is_active, + timeframe_to_prev_date) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_exchange_credentials) -from freqtrade.exchange.exchange import amount_to_contract_precision from freqtrade.resolvers.exchange_resolver import ExchangeResolver from freqtrade.util import dt_now, dt_ts from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_patched_exchange, @@ -28,7 +24,7 @@ from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_pat # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit'] +EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] get_entry_rate_data = [ ('other', 20, 19, 10, 0.0, 20), # Full ask side @@ -287,87 +283,6 @@ def test_validate_order_time_in_force(default_conf, mocker, caplog): ex.validate_order_time_in_force(tif2) -@pytest.mark.parametrize("amount,precision_mode,precision,expected", [ - (2.34559, 2, 4, 2.3455), - (2.34559, 2, 5, 2.34559), - (2.34559, 2, 3, 2.345), - (2.9999, 2, 3, 2.999), - (2.9909, 2, 3, 2.990), - (2.9909, 2, 0, 2), - (29991.5555, 2, 0, 29991), - (29991.5555, 2, -1, 29990), - (29991.5555, 2, -2, 29900), - # Tests for Tick-size - (2.34559, 4, 0.0001, 2.3455), - (2.34559, 4, 0.00001, 2.34559), - (2.34559, 4, 0.001, 2.345), - (2.9999, 4, 0.001, 2.999), - (2.9909, 4, 0.001, 2.990), - (2.9909, 4, 0.005, 2.99), - (2.9999, 4, 0.005, 2.995), -]) -def test_amount_to_precision(amount, precision_mode, precision, expected,): - """ - Test rounds down - """ - # digits counting mode - # DECIMAL_PLACES = 2 - # SIGNIFICANT_DIGITS = 3 - # TICK_SIZE = 4 - - assert amount_to_precision(amount, precision, precision_mode) == expected - - -@pytest.mark.parametrize("price,precision_mode,precision,expected,rounding_mode", [ - # Tests for DECIMAL_PLACES, ROUND_UP - (2.34559, 2, 4, 2.3456, ROUND_UP), - (2.34559, 2, 5, 2.34559, ROUND_UP), - (2.34559, 2, 3, 2.346, ROUND_UP), - (2.9999, 2, 3, 3.000, ROUND_UP), - (2.9909, 2, 3, 2.991, ROUND_UP), - # Tests for DECIMAL_PLACES, ROUND - (2.345600000000001, DECIMAL_PLACES, 4, 2.3456, ROUND), - (2.345551, DECIMAL_PLACES, 4, 2.3456, ROUND), - (2.49, DECIMAL_PLACES, 0, 2., ROUND), - (2.51, DECIMAL_PLACES, 0, 3., ROUND), - (5.1, DECIMAL_PLACES, -1, 10., ROUND), - (4.9, DECIMAL_PLACES, -1, 0., ROUND), - # Tests for TICK_SIZE, ROUND_UP - (2.34559, TICK_SIZE, 0.0001, 2.3456, ROUND_UP), - (2.34559, TICK_SIZE, 0.00001, 2.34559, ROUND_UP), - (2.34559, TICK_SIZE, 0.001, 2.346, ROUND_UP), - (2.9999, TICK_SIZE, 0.001, 3.000, ROUND_UP), - (2.9909, TICK_SIZE, 0.001, 2.991, ROUND_UP), - (2.9909, TICK_SIZE, 0.005, 2.995, ROUND_UP), - (2.9973, TICK_SIZE, 0.005, 3.0, ROUND_UP), - (2.9977, TICK_SIZE, 0.005, 3.0, ROUND_UP), - (234.43, TICK_SIZE, 0.5, 234.5, ROUND_UP), - (234.53, TICK_SIZE, 0.5, 235.0, ROUND_UP), - (0.891534, TICK_SIZE, 0.0001, 0.8916, ROUND_UP), - (64968.89, TICK_SIZE, 0.01, 64968.89, ROUND_UP), - (0.000000003483, TICK_SIZE, 1e-12, 0.000000003483, ROUND_UP), - # Tests for TICK_SIZE, ROUND - (2.49, TICK_SIZE, 1., 2., ROUND), - (2.51, TICK_SIZE, 1., 3., ROUND), - (2.000000051, TICK_SIZE, 0.0000001, 2.0000001, ROUND), - (2.000000049, TICK_SIZE, 0.0000001, 2., ROUND), - (2.9909, TICK_SIZE, 0.005, 2.990, ROUND), - (2.9973, TICK_SIZE, 0.005, 2.995, ROUND), - (2.9977, TICK_SIZE, 0.005, 3.0, ROUND), - (234.24, TICK_SIZE, 0.5, 234., ROUND), - (234.26, TICK_SIZE, 0.5, 234.5, ROUND), - # Tests for TRUNCATTE - (2.34559, 2, 4, 2.3455, TRUNCATE), - (2.34559, 2, 5, 2.34559, TRUNCATE), - (2.34559, 2, 3, 2.345, TRUNCATE), - (2.9999, 2, 3, 2.999, TRUNCATE), - (2.9909, 2, 3, 2.990, TRUNCATE), -]) -def test_price_to_precision(price, precision_mode, precision, expected, rounding_mode): - assert price_to_precision( - price, precision, precision_mode, rounding_mode=rounding_mode) == expected - - @pytest.mark.parametrize("price,precision_mode,precision,expected", [ (2.34559, 2, 4, 0.0001), (2.34559, 2, 5, 0.00001), @@ -1397,8 +1312,11 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice, leverage=3.0 ) - assert exchange._set_leverage.call_count == 1 - assert exchange.set_margin_mode.call_count == 1 + if exchange_name != 'okx': + assert exchange._set_leverage.call_count == 1 + assert exchange.set_margin_mode.call_count == 1 + else: + assert api_mock.set_leverage.call_count == 1 assert order['amount'] == 0.01 @@ -1529,6 +1447,7 @@ def test_buy_considers_time_in_force(default_conf, mocker, exchange_name): assert 'id' in order assert 'info' in order + assert order['status'] == 'open' assert order['id'] == order_id assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type @@ -1762,7 +1681,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) mocker.patch(f'{EXMS}.exchange_has', return_value=True) - start_time = datetime.now(timezone.utc) - timedelta(days=5) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + expected = 1 + if exchange_name == 'bybit': + expected = 3 exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # Not available in dry-run @@ -1772,10 +1694,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) res = exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected assert api_mock.fetch_open_orders.call_count == 0 assert api_mock.fetch_closed_orders.call_count == 0 - assert len(res) == 2 + assert len(res) == 2 * expected res = exchange.fetch_orders('mocked', start_time) @@ -1789,13 +1711,17 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): if endpoint == 'fetchOpenOrders': return True + if exchange_name == 'okx': + # Special OKX case is tested separately + return + mocker.patch(f'{EXMS}.exchange_has', has_resp) # happy path without fetchOrders - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) assert api_mock.fetch_orders.call_count == 0 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected mocker.patch(f'{EXMS}.exchange_has', return_value=True) @@ -1808,11 +1734,11 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_open_orders.reset_mock() api_mock.fetch_closed_orders.reset_mock() - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected def test_fetch_trading_fees(default_conf, mocker): @@ -2129,7 +2055,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ) # Required candles candles = (end_ts - start_ts) / 300_000 - exp = candles // exchange.ohlcv_candle_limit('5m', CandleType.SPOT) + 1 + exp = candles // exchange.ohlcv_candle_limit('5m', candle_type, start_ts) + 1 # Depending on the exchange, this should be called between 1 and 6 times. assert exchange._api_async.fetch_ohlcv.call_count == exp @@ -3207,25 +3133,28 @@ def test_cancel_stoploss_order(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): default_conf['dry_run'] = False + mock_prefix = 'freqtrade.exchange.gate.Gate' + if exchange_name == 'okx': + mock_prefix = 'freqtrade.exchange.okx.Okx' mocker.patch(f'{EXMS}.fetch_stoploss_order', return_value={'for': 123}) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', return_value={'for': 123}) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', return_value={'for': 123}) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) res = {'fee': {}, 'status': 'canceled', 'amount': 1234} mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value=res) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value=res) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value=res) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == res mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value='canceled') - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value='canceled') + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value='canceled') # Fall back to fetch_stoploss_order co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == {'for': 123} exc = InvalidOrderException("") mocker.patch(f'{EXMS}.fetch_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', side_effect=exc) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co['amount'] == 555 assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} @@ -3233,7 +3162,7 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): with pytest.raises(InvalidOrderException): exc = InvalidOrderException("Did not find order") mocker.patch(f'{EXMS}.cancel_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', side_effect=exc) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=123) @@ -3308,8 +3237,14 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name): api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value={'id': '123', 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == {'id': '123', 'symbol': 'TKN/BTC'} + res = {'id': '123', 'symbol': 'TKN/BTC'} + if exchange_name == 'okx': + res = {'id': '123', 'symbol': 'TKN/BTC', 'type': 'stoploss'} + assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == res + if exchange_name == 'okx': + # Tested separately. + return with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) @@ -3629,6 +3564,8 @@ def test_get_markets_error(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): + if exchange_name == 'okx': + pytest.skip("Tested separately for okx") exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) timeframes = ('1m', '5m', '1h') expected = exchange._ft_has['ohlcv_candle_limit'] @@ -3640,96 +3577,6 @@ def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): assert exchange.ohlcv_candle_limit(timeframe, CandleType.SPOT) == expected -def test_timeframe_to_minutes(): - assert timeframe_to_minutes("5m") == 5 - assert timeframe_to_minutes("10m") == 10 - assert timeframe_to_minutes("1h") == 60 - assert timeframe_to_minutes("1d") == 1440 - - -def test_timeframe_to_seconds(): - assert timeframe_to_seconds("5m") == 300 - assert timeframe_to_seconds("10m") == 600 - assert timeframe_to_seconds("1h") == 3600 - assert timeframe_to_seconds("1d") == 86400 - - -def test_timeframe_to_msecs(): - assert timeframe_to_msecs("5m") == 300000 - assert timeframe_to_msecs("10m") == 600000 - assert timeframe_to_msecs("1h") == 3600000 - assert timeframe_to_msecs("1d") == 86400000 - - -def test_timeframe_to_prev_date(): - # 2019-08-12 13:22:08 - date = datetime.fromtimestamp(1565616128, tz=timezone.utc) - - tf_list = [ - # 5m -> 2019-08-12 13:20:00 - ("5m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), - # 10m -> 2019-08-12 13:20:00 - ("10m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), - # 1h -> 2019-08-12 13:00:00 - ("1h", datetime(2019, 8, 12, 13, 00, 0, tzinfo=timezone.utc)), - # 2h -> 2019-08-12 12:00:00 - ("2h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), - # 4h -> 2019-08-12 12:00:00 - ("4h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), - # 1d -> 2019-08-12 00:00:00 - ("1d", datetime(2019, 8, 12, 00, 00, 0, tzinfo=timezone.utc)), - ] - for interval, result in tf_list: - assert timeframe_to_prev_date(interval, date) == result - - date = datetime.now(tz=timezone.utc) - assert timeframe_to_prev_date("5m") < date - # Does not round - time = datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc) - assert timeframe_to_prev_date('5m', time) == time - time = datetime(2019, 8, 12, 13, 0, 0, tzinfo=timezone.utc) - assert timeframe_to_prev_date('1h', time) == time - - -def test_timeframe_to_next_date(): - # 2019-08-12 13:22:08 - date = datetime.fromtimestamp(1565616128, tz=timezone.utc) - tf_list = [ - # 5m -> 2019-08-12 13:25:00 - ("5m", datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc)), - # 10m -> 2019-08-12 13:30:00 - ("10m", datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc)), - # 1h -> 2019-08-12 14:00:00 - ("1h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), - # 2h -> 2019-08-12 14:00:00 - ("2h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), - # 4h -> 2019-08-12 14:00:00 - ("4h", datetime(2019, 8, 12, 16, 00, 0, tzinfo=timezone.utc)), - # 1d -> 2019-08-13 00:00:00 - ("1d", datetime(2019, 8, 13, 0, 0, 0, tzinfo=timezone.utc)), - ] - - for interval, result in tf_list: - assert timeframe_to_next_date(interval, date) == result - - date = datetime.now(tz=timezone.utc) - assert timeframe_to_next_date("5m") > date - - date = datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc) - assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5) - - -def test_date_minus_candles(): - - date = datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc) - - assert date_minus_candles("5m", 3, date) == date - timedelta(minutes=15) - assert date_minus_candles("5m", 5, date) == date - timedelta(minutes=25) - assert date_minus_candles("1m", 6, date) == date - timedelta(minutes=6) - assert date_minus_candles("1h", 3, date) == date - timedelta(hours=3, minutes=25) - assert date_minus_candles("1h", 3) == timeframe_to_prev_date('1h') - timedelta(hours=3) - - @pytest.mark.parametrize( "market_symbol,base,quote,exchange,spot,margin,futures,trademode,add_dict,expected_result", [ @@ -4623,20 +4470,6 @@ def test_amount_to_contract_precision( assert result_size == expected_fut -@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [ - (1.17, 1.0, 4, 0.01, 1.17), # Tick size - (1.17, 1.0, 2, 0.01, 1.17), # - (1.16, 1.0, 4, 0.01, 1.16), # - (1.16, 1.0, 2, 0.01, 1.16), # - (1.13, 1.0, 2, 0.01, 1.13), # - (10.988, 1.0, 2, 10, 10), - (10.988, 1.0, 4, 10, 10), -]) -def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected): - res = amount_to_contract_precision(amount, precision, precision_mode, contract_size) - assert pytest.approx(res) == expected - - @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [ # Bittrex ('bittrex', 2.0, False, 'spot', None), diff --git a/tests/exchange/test_exchange_utils.py b/tests/exchange/test_exchange_utils.py index db206ab98..377514468 100644 --- a/tests/exchange/test_exchange_utils.py +++ b/tests/exchange/test_exchange_utils.py @@ -1,9 +1,16 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name +from datetime import datetime, timedelta, timezone import pytest +from ccxt import (DECIMAL_PLACES, ROUND, ROUND_DOWN, ROUND_UP, SIGNIFICANT_DIGITS, TICK_SIZE, + TRUNCATE) from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (amount_to_contract_precision, amount_to_precision, + date_minus_candles, price_to_precision, timeframe_to_minutes, + timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, + timeframe_to_seconds) from freqtrade.exchange.check_exchange import check_exchange from tests.conftest import log_has_re @@ -83,3 +90,239 @@ def test_check_exchange(default_conf, caplog) -> None: with pytest.raises(OperationalException, match=r'This command requires a configured exchange.*'): check_exchange(default_conf) + + +def test_date_minus_candles(): + + date = datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc) + + assert date_minus_candles("5m", 3, date) == date - timedelta(minutes=15) + assert date_minus_candles("5m", 5, date) == date - timedelta(minutes=25) + assert date_minus_candles("1m", 6, date) == date - timedelta(minutes=6) + assert date_minus_candles("1h", 3, date) == date - timedelta(hours=3, minutes=25) + assert date_minus_candles("1h", 3) == timeframe_to_prev_date('1h') - timedelta(hours=3) + + +def test_timeframe_to_minutes(): + assert timeframe_to_minutes("5m") == 5 + assert timeframe_to_minutes("10m") == 10 + assert timeframe_to_minutes("1h") == 60 + assert timeframe_to_minutes("1d") == 1440 + + +def test_timeframe_to_seconds(): + assert timeframe_to_seconds("5m") == 300 + assert timeframe_to_seconds("10m") == 600 + assert timeframe_to_seconds("1h") == 3600 + assert timeframe_to_seconds("1d") == 86400 + + +def test_timeframe_to_msecs(): + assert timeframe_to_msecs("5m") == 300000 + assert timeframe_to_msecs("10m") == 600000 + assert timeframe_to_msecs("1h") == 3600000 + assert timeframe_to_msecs("1d") == 86400000 + + +def test_timeframe_to_prev_date(): + # 2019-08-12 13:22:08 + date = datetime.fromtimestamp(1565616128, tz=timezone.utc) + + tf_list = [ + # 5m -> 2019-08-12 13:20:00 + ("5m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), + # 10m -> 2019-08-12 13:20:00 + ("10m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), + # 1h -> 2019-08-12 13:00:00 + ("1h", datetime(2019, 8, 12, 13, 00, 0, tzinfo=timezone.utc)), + # 2h -> 2019-08-12 12:00:00 + ("2h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), + # 4h -> 2019-08-12 12:00:00 + ("4h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), + # 1d -> 2019-08-12 00:00:00 + ("1d", datetime(2019, 8, 12, 00, 00, 0, tzinfo=timezone.utc)), + ] + for interval, result in tf_list: + assert timeframe_to_prev_date(interval, date) == result + + date = datetime.now(tz=timezone.utc) + assert timeframe_to_prev_date("5m") < date + # Does not round + time = datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc) + assert timeframe_to_prev_date('5m', time) == time + time = datetime(2019, 8, 12, 13, 0, 0, tzinfo=timezone.utc) + assert timeframe_to_prev_date('1h', time) == time + + +def test_timeframe_to_next_date(): + # 2019-08-12 13:22:08 + date = datetime.fromtimestamp(1565616128, tz=timezone.utc) + tf_list = [ + # 5m -> 2019-08-12 13:25:00 + ("5m", datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc)), + # 10m -> 2019-08-12 13:30:00 + ("10m", datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc)), + # 1h -> 2019-08-12 14:00:00 + ("1h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), + # 2h -> 2019-08-12 14:00:00 + ("2h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), + # 4h -> 2019-08-12 14:00:00 + ("4h", datetime(2019, 8, 12, 16, 00, 0, tzinfo=timezone.utc)), + # 1d -> 2019-08-13 00:00:00 + ("1d", datetime(2019, 8, 13, 0, 0, 0, tzinfo=timezone.utc)), + ] + + for interval, result in tf_list: + assert timeframe_to_next_date(interval, date) == result + + date = datetime.now(tz=timezone.utc) + assert timeframe_to_next_date("5m") > date + + date = datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc) + assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5) + + +@pytest.mark.parametrize("amount,precision_mode,precision,expected", [ + (2.34559, DECIMAL_PLACES, 4, 2.3455), + (2.34559, DECIMAL_PLACES, 5, 2.34559), + (2.34559, DECIMAL_PLACES, 3, 2.345), + (2.9999, DECIMAL_PLACES, 3, 2.999), + (2.9909, DECIMAL_PLACES, 3, 2.990), + (2.9909, DECIMAL_PLACES, 0, 2), + (29991.5555, DECIMAL_PLACES, 0, 29991), + (29991.5555, DECIMAL_PLACES, -1, 29990), + (29991.5555, DECIMAL_PLACES, -2, 29900), + # Tests for + (2.34559, SIGNIFICANT_DIGITS, 4, 2.345), + (2.34559, SIGNIFICANT_DIGITS, 5, 2.3455), + (2.34559, SIGNIFICANT_DIGITS, 3, 2.34), + (2.9999, SIGNIFICANT_DIGITS, 3, 2.99), + (2.9909, SIGNIFICANT_DIGITS, 3, 2.99), + (0.0000077723, SIGNIFICANT_DIGITS, 5, 0.0000077723), + (0.0000077723, SIGNIFICANT_DIGITS, 3, 0.00000777), + (0.0000077723, SIGNIFICANT_DIGITS, 1, 0.000007), + # Tests for Tick-size + (2.34559, TICK_SIZE, 0.0001, 2.3455), + (2.34559, TICK_SIZE, 0.00001, 2.34559), + (2.34559, TICK_SIZE, 0.001, 2.345), + (2.9999, TICK_SIZE, 0.001, 2.999), + (2.9909, TICK_SIZE, 0.001, 2.990), + (2.9909, TICK_SIZE, 0.005, 2.99), + (2.9999, TICK_SIZE, 0.005, 2.995), +]) +def test_amount_to_precision(amount, precision_mode, precision, expected,): + """ + Test rounds down + """ + # digits counting mode + # DECIMAL_PLACES = 2 + # SIGNIFICANT_DIGITS = 3 + # TICK_SIZE = 4 + + assert amount_to_precision(amount, precision, precision_mode) == expected + + +@pytest.mark.parametrize("price,precision_mode,precision,expected,rounding_mode", [ + # Tests for DECIMAL_PLACES, ROUND_UP + (2.34559, DECIMAL_PLACES, 4, 2.3456, ROUND_UP), + (2.34559, DECIMAL_PLACES, 5, 2.34559, ROUND_UP), + (2.34559, DECIMAL_PLACES, 3, 2.346, ROUND_UP), + (2.9999, DECIMAL_PLACES, 3, 3.000, ROUND_UP), + (2.9909, DECIMAL_PLACES, 3, 2.991, ROUND_UP), + (2.9901, DECIMAL_PLACES, 3, 2.991, ROUND_UP), + (2.34559, DECIMAL_PLACES, 5, 2.34559, ROUND_DOWN), + (2.34559, DECIMAL_PLACES, 4, 2.3455, ROUND_DOWN), + (2.9901, DECIMAL_PLACES, 3, 2.990, ROUND_DOWN), + (0.00299, DECIMAL_PLACES, 3, 0.002, ROUND_DOWN), + # Tests for DECIMAL_PLACES, ROUND + (2.345600000000001, DECIMAL_PLACES, 4, 2.3456, ROUND), + (2.345551, DECIMAL_PLACES, 4, 2.3456, ROUND), + (2.49, DECIMAL_PLACES, 0, 2., ROUND), + (2.51, DECIMAL_PLACES, 0, 3., ROUND), + (5.1, DECIMAL_PLACES, -1, 10., ROUND), + (4.9, DECIMAL_PLACES, -1, 0., ROUND), + (0.000007222, SIGNIFICANT_DIGITS, 1, 0.000007, ROUND), + (0.000007222, SIGNIFICANT_DIGITS, 2, 0.0000072, ROUND), + (0.000007777, SIGNIFICANT_DIGITS, 2, 0.0000078, ROUND), + # Tests for TICK_SIZE, ROUND_UP + (2.34559, TICK_SIZE, 0.0001, 2.3456, ROUND_UP), + (2.34559, TICK_SIZE, 0.00001, 2.34559, ROUND_UP), + (2.34559, TICK_SIZE, 0.001, 2.346, ROUND_UP), + (2.9999, TICK_SIZE, 0.001, 3.000, ROUND_UP), + (2.9909, TICK_SIZE, 0.001, 2.991, ROUND_UP), + (2.9909, TICK_SIZE, 0.001, 2.990, ROUND_DOWN), + (2.9909, TICK_SIZE, 0.005, 2.995, ROUND_UP), + (2.9973, TICK_SIZE, 0.005, 3.0, ROUND_UP), + (2.9977, TICK_SIZE, 0.005, 3.0, ROUND_UP), + (234.43, TICK_SIZE, 0.5, 234.5, ROUND_UP), + (234.43, TICK_SIZE, 0.5, 234.0, ROUND_DOWN), + (234.53, TICK_SIZE, 0.5, 235.0, ROUND_UP), + (234.53, TICK_SIZE, 0.5, 234.5, ROUND_DOWN), + (0.891534, TICK_SIZE, 0.0001, 0.8916, ROUND_UP), + (64968.89, TICK_SIZE, 0.01, 64968.89, ROUND_UP), + (0.000000003483, TICK_SIZE, 1e-12, 0.000000003483, ROUND_UP), + # Tests for TICK_SIZE, ROUND + (2.49, TICK_SIZE, 1., 2., ROUND), + (2.51, TICK_SIZE, 1., 3., ROUND), + (2.000000051, TICK_SIZE, 0.0000001, 2.0000001, ROUND), + (2.000000049, TICK_SIZE, 0.0000001, 2., ROUND), + (2.9909, TICK_SIZE, 0.005, 2.990, ROUND), + (2.9973, TICK_SIZE, 0.005, 2.995, ROUND), + (2.9977, TICK_SIZE, 0.005, 3.0, ROUND), + (234.24, TICK_SIZE, 0.5, 234., ROUND), + (234.26, TICK_SIZE, 0.5, 234.5, ROUND), + # Tests for TRUNCATTE + (2.34559, DECIMAL_PLACES, 4, 2.3455, TRUNCATE), + (2.34559, DECIMAL_PLACES, 5, 2.34559, TRUNCATE), + (2.34559, DECIMAL_PLACES, 3, 2.345, TRUNCATE), + (2.9999, DECIMAL_PLACES, 3, 2.999, TRUNCATE), + (2.9909, DECIMAL_PLACES, 3, 2.990, TRUNCATE), + (2.9909, TICK_SIZE, 0.001, 2.990, TRUNCATE), + (2.9909, TICK_SIZE, 0.01, 2.99, TRUNCATE), + (2.9909, TICK_SIZE, 0.1, 2.9, TRUNCATE), + # Tests for Significant + (2.34559, SIGNIFICANT_DIGITS, 4, 2.345, TRUNCATE), + (2.34559, SIGNIFICANT_DIGITS, 5, 2.3455, TRUNCATE), + (2.34559, SIGNIFICANT_DIGITS, 3, 2.34, TRUNCATE), + (2.9999, SIGNIFICANT_DIGITS, 3, 2.99, TRUNCATE), + (2.9909, SIGNIFICANT_DIGITS, 2, 2.9, TRUNCATE), + (0.00000777, SIGNIFICANT_DIGITS, 2, 0.0000077, TRUNCATE), + (0.00000729, SIGNIFICANT_DIGITS, 2, 0.0000072, TRUNCATE), + # ROUND + (722.2, SIGNIFICANT_DIGITS, 1, 700.0, ROUND), + (790.2, SIGNIFICANT_DIGITS, 1, 800.0, ROUND), + (722.2, SIGNIFICANT_DIGITS, 2, 720.0, ROUND), + (722.2, SIGNIFICANT_DIGITS, 1, 800.0, ROUND_UP), + (722.2, SIGNIFICANT_DIGITS, 2, 730.0, ROUND_UP), + (777.7, SIGNIFICANT_DIGITS, 2, 780.0, ROUND_UP), + (777.7, SIGNIFICANT_DIGITS, 3, 778.0, ROUND_UP), + (722.2, SIGNIFICANT_DIGITS, 1, 700.0, ROUND_DOWN), + (722.2, SIGNIFICANT_DIGITS, 2, 720.0, ROUND_DOWN), + (777.7, SIGNIFICANT_DIGITS, 2, 770.0, ROUND_DOWN), + (777.7, SIGNIFICANT_DIGITS, 3, 777.0, ROUND_DOWN), + + (0.000007222, SIGNIFICANT_DIGITS, 1, 0.000008, ROUND_UP), + (0.000007222, SIGNIFICANT_DIGITS, 2, 0.0000073, ROUND_UP), + (0.000007777, SIGNIFICANT_DIGITS, 2, 0.0000078, ROUND_UP), + (0.000007222, SIGNIFICANT_DIGITS, 1, 0.000007, ROUND_DOWN), + (0.000007222, SIGNIFICANT_DIGITS, 2, 0.0000072, ROUND_DOWN), + (0.000007777, SIGNIFICANT_DIGITS, 2, 0.0000077, ROUND_DOWN), +]) +def test_price_to_precision(price, precision_mode, precision, expected, rounding_mode): + assert price_to_precision( + price, precision, precision_mode, rounding_mode=rounding_mode) == expected + + +@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [ + (1.17, 1.0, 4, 0.01, 1.17), # Tick size + (1.17, 1.0, 2, 0.01, 1.17), # + (1.16, 1.0, 4, 0.01, 1.16), # + (1.16, 1.0, 2, 0.01, 1.16), # + (1.13, 1.0, 2, 0.01, 1.13), # + (10.988, 1.0, 2, 10, 10), + (10.988, 1.0, 4, 10, 10), +]) +def test_amount_to_contract_precision_standalone(amount, precision, precision_mode, contract_size, + expected): + res = amount_to_contract_precision(amount, precision, precision_mode, contract_size) + assert pytest.approx(res) == expected diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index e8f059118..736c630e0 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -618,3 +618,70 @@ def test__get_stop_params_okx(mocker, default_conf): assert params['tdMode'] == 'isolated' assert params['posSide'] == 'net' + + +def test_fetch_orders_okx(default_conf, mocker, limit_order): + + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[ + limit_order['buy'], + limit_order['sell'], + ]) + api_mock.fetch_open_orders = MagicMock(return_value=[limit_order['buy']]) + api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + # Not available in dry-run + assert exchange.fetch_orders('mocked', start_time) == [] + assert api_mock.fetch_orders.call_count == 0 + default_conf['dry_run'] = False + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + + def has_resp(_, endpoint): + if endpoint == 'fetchOrders': + return False + if endpoint == 'fetchClosedOrders': + return True + if endpoint == 'fetchOpenOrders': + return True + + mocker.patch(f'{EXMS}.exchange_has', has_resp) + + history_params = {'method': 'privateGetTradeOrdersHistoryArchive'} + + # happy path without fetchOrders + exchange.fetch_orders('mocked', start_time) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params + + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + # regular closed_orders endpoint only has history for 7 days. + exchange.fetch_orders('mocked', datetime.now(timezone.utc) - timedelta(days=6)) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 1 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + + # Unhappy path - first fetch-orders call fails. + api_mock.fetch_orders = MagicMock(side_effect=ccxt.NotSupported()) + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + exchange.fetch_orders('mocked', start_time) + + assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index 45da16dd5..c5f59ee0e 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -234,12 +234,12 @@ EXCHANGES = { "orderId": "1274754916287346280", "orderLinkId": "1666798627015730", "symbol": "SOLUSDT", - "createTime": "1674493798550", - "orderPrice": "15.5", - "orderQty": "1.1", - "orderType": "LIMIT", - "side": "BUY", - "status": "NEW", + "createdTime": "1674493798550", + "price": "15.5", + "qty": "1.1", + "orderType": "Limit", + "side": "Buy", + "orderStatus": "New", "timeInForce": "GTC", "accountId": "5555555", "execQty": "0", diff --git a/tests/exchange_online/test_ccxt_compat.py b/tests/exchange_online/test_ccxt_compat.py index e33875416..aa3dfdfae 100644 --- a/tests/exchange_online/test_ccxt_compat.py +++ b/tests/exchange_online/test_ccxt_compat.py @@ -58,8 +58,10 @@ class TestCCXTExchange: def test_ccxt_order_parse(self, exchange: EXCHANGE_FIXTURE_TYPE): exch, exchange_name = exchange if orders := EXCHANGES[exchange_name].get('sample_order'): + pair = 'SOL/USDT' for order in orders: - po = exch._api.parse_order(order) + market = exch._api.markets[pair] + po = exch._api.parse_order(order, market) assert isinstance(po['id'], str) assert po['id'] is not None if len(order.keys()) < 5: @@ -74,7 +76,7 @@ class TestCCXTExchange: if po['average'] is not None: assert isinstance(po['average'], float) assert po['average'] == 15.5 - assert po['symbol'] == 'SOL/USDT' + assert po['symbol'] == pair assert isinstance(po['amount'], float) assert po['amount'] == 1.1 assert isinstance(po['status'], str) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 8ab2c75da..ca4749747 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -1,7 +1,9 @@ import shutil from pathlib import Path +from unittest.mock import patch +import pandas as pd import pytest from freqtrade.configuration import TimeRange @@ -135,3 +137,111 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co match=r'Historic predictions not found.*' ): freqai.dd.get_timerange_from_live_historic_predictions() + + +def test_set_initial_return_values(mocker, freqai_conf): + """ + Simple test of the set initial return values that ensures + we are concatening and ffilling values properly. + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-08-30" + end_x_plus_5 = "2023-09-03" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert (hist_pred_df['date_pred'].iloc[-1] == + pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)) + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == + hist_pred_df.tail(len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_not_called() + + +def test_set_initial_return_values_warning(mocker, freqai_conf): + """ + Simple test of set_initial_return_values that hits the warning + associated with leaving a FreqAI bot offline so long that the + exchange candles have no common date with the historic predictions + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-09-01" + end_x_plus_5 = "2023-09-05" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1) + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == hist_pred_df.tail( + len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_called() diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index ce26e836e..99e1ec812 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -9,7 +9,7 @@ import pytest from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import get_timerange -from freqtrade.enums import ExitType, TradingMode +from freqtrade.enums import ExitType from freqtrade.optimize.backtesting import Backtesting from freqtrade.util.datetime_helpers import dt_utc from tests.conftest import EXMS, patch_exchange @@ -103,18 +103,21 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) mocker.patch(f"{EXMS}.get_max_leverage", return_value=10) + mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1)) patch_exchange(mocker) default_conf.update({ "stake_amount": 100.0, "dry_run_wallet": 1000.0, "strategy": "StrategyTestV3", + "trading_mode": "futures", + "margin_mode": "isolated", }) + default_conf['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}] backtesting = Backtesting(default_conf) - backtesting.trading_mode = TradingMode.FUTURES backtesting._can_short = True backtesting._set_strategy(backtesting.strategylist[0]) - pair = 'XRP/USDT' + pair = 'XRP/USDT:USDT' row = [ pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), 2.1, # Open @@ -130,12 +133,12 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera ] backtesting.strategy.leverage = MagicMock(return_value=leverage) trade = backtesting._enter_trade(pair, row=row, direction='long') - trade.orders[0].close_bt_order(row[0], trade) assert trade assert pytest.approx(trade.stake_amount) == 100.0 assert pytest.approx(trade.amount) == 47.61904762 * leverage assert len(trade.orders) == 1 backtesting.strategy.adjust_trade_position = MagicMock(return_value=None) + assert pytest.approx(trade.liquidation_price) == (0.10278333 if leverage == 1 else 1.2122249) trade = backtesting._get_adjust_trade_entry_for_candle(trade, row) assert trade @@ -151,6 +154,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert pytest.approx(trade.stake_amount) == 200.0 assert pytest.approx(trade.amount) == 95.23809524 * leverage assert len(trade.orders) == 2 + assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) # Reduce by more than amount - no change to trade. backtesting.strategy.adjust_trade_position = MagicMock(return_value=-500) @@ -162,6 +166,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert pytest.approx(trade.amount) == 95.23809524 * leverage assert len(trade.orders) == 2 assert trade.nr_of_successful_entries == 2 + assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) # Reduce position by 50 backtesting.strategy.adjust_trade_position = MagicMock(return_value=-100) @@ -173,6 +178,7 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert len(trade.orders) == 3 assert trade.nr_of_successful_entries == 2 assert trade.nr_of_successful_exits == 1 + assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) # Adjust below minimum backtesting.strategy.adjust_trade_position = MagicMock(return_value=-99) @@ -184,3 +190,4 @@ def test_backtest_position_adjustment_detailed(default_conf, fee, mocker, levera assert len(trade.orders) == 3 assert trade.nr_of_successful_entries == 2 assert trade.nr_of_successful_exits == 1 + assert pytest.approx(trade.liquidation_price) == (0.1038916 if leverage == 1 else 1.2127791) diff --git a/tests/optimize/test_recursive_analysis.py b/tests/optimize/test_recursive_analysis.py new file mode 100644 index 000000000..f025f5b76 --- /dev/null +++ b/tests/optimize/test_recursive_analysis.py @@ -0,0 +1,188 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +from copy import deepcopy +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pytest + +from freqtrade.commands.optimize_commands import start_recursive_analysis +from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.recursive_analysis import RecursiveAnalysis +from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions +from tests.conftest import get_args, log_has_re, patch_exchange + + +@pytest.fixture +def recursive_conf(default_conf_usdt): + default_conf_usdt['timerange'] = '20220101-20220501' + + default_conf_usdt['strategy_path'] = str( + Path(__file__).parent.parent / "strategy/strats") + default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue' + default_conf_usdt['pairs'] = ['UNITTEST/USDT'] + default_conf_usdt['startup_candle'] = [100] + return default_conf_usdt + + +def test_start_recursive_analysis(mocker): + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_recursive_issue", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC", + "--timerange", + "20220101-20220201" + ] + pargs = get_args(args) + pargs['config'] = None + + start_recursive_analysis(pargs) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + + # Missing timerange + args = [ + "recursive-analysis", + "--strategy", + "strategy_test_v3_with_recursive_bias", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy/strats"), + "--pairs", + "UNITTEST/BTC" + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Please set a timerange\..*"): + start_recursive_analysis(pargs) + + +def test_recursive_helper_no_strategy_defined(recursive_conf): + conf = deepcopy(recursive_conf) + conf['pairs'] = ['UNITTEST/USDT'] + del conf['strategy'] + with pytest.raises(OperationalException, + match=r"No Strategy specified"): + RecursiveAnalysisSubFunctions.start(conf) + + +def test_recursive_helper_start(recursive_conf, mocker) -> None: + single_mock = MagicMock() + text_table_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions', + initialize_single_recursive_analysis=single_mock, + text_table_recursive_analysis_instances=text_table_mock, + ) + RecursiveAnalysisSubFunctions.start(recursive_conf) + assert single_mock.call_count == 1 + assert text_table_mock.call_count == 1 + + single_mock.reset_mock() + text_table_mock.reset_mock() + + +def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf): + dict_diff = dict() + dict_diff['rsi'] = {} + dict_diff['rsi'][100] = "0.078%" + + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } + + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) + + # check row contents for a try that has too few signals + assert data[0][0] == 'rsi' + assert data[0][1] == '0.078%' + assert len(data[0]) == 2 + + # now check when there is no issue + dict_diff = dict() + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.dict_recursive = dict_diff + table, headers, data = (RecursiveAnalysisSubFunctions. + text_table_recursive_analysis_instances([instance])) + assert len(data) == 0 + + +def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog): + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + recursive_conf['pairs'] = ['UNITTEST/BTC'] + + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + start_mock = mocker.patch('freqtrade.optimize.recursive_analysis.RecursiveAnalysis.start') + strategy_obj = { + 'name': "strategy_test_v3_recursive_issue", + 'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py") + } + + instance = RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis( + recursive_conf, strategy_obj) + assert log_has_re(r"Recursive test of .* started\.", caplog) + assert start_mock.call_count == 1 + + assert instance.strategy_obj['name'] == "strategy_test_v3_recursive_issue" + + +@pytest.mark.parametrize('scenario', [ + 'no_bias', 'bias1', 'bias2' +]) +def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None: + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) + recursive_conf['pairs'] = ['UNITTEST/BTC'] + + recursive_conf['timeframe'] = '5m' + recursive_conf['timerange'] = '20180119-20180122' + recursive_conf['startup_candle'] = [100] + + # Patch scenario Parameter to allow for easy selection + mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file', + return_value={ + 'params': { + "buy": { + "scenario": scenario + } + } + }) + + strategy_obj = {'name': "strategy_test_v3_recursive_issue"} + instance = RecursiveAnalysis(recursive_conf, strategy_obj) + instance.start() + # Assert init correct + assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog) + + if scenario == "bias2": + assert log_has_re("=> found lookahead in indicator rsi", caplog) + diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", ""))) + # check non-biased strategy + if scenario == "no_bias": + assert diff_pct < 0.01 + # check biased strategy + elif scenario in ("bias1", "bias2"): + assert diff_pct >= 0.01 diff --git a/tests/persistence/test_migrations.py b/tests/persistence/test_migrations.py index 13b3f89bf..1e87d3940 100644 --- a/tests/persistence/test_migrations.py +++ b/tests/persistence/test_migrations.py @@ -15,6 +15,7 @@ from freqtrade.persistence import Trade, init_db from freqtrade.persistence.base import ModelBase from freqtrade.persistence.migrations import get_last_sequence_ids, set_sequence_ids from freqtrade.persistence.models import PairLock +from freqtrade.persistence.trade_model import Order from tests.conftest import log_has @@ -217,6 +218,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog): {amount}, 0, {amount * 0.00258580} + ), + ( + -- Order without reference trade + 2, + 'buy', + 'ETC/BTC', + 1, + 'dry_buy_order55', + 'canceled', + 'ETC/BTC', + 'limit', + 'buy', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} ) """ engine = create_engine('sqlite://') @@ -238,9 +256,10 @@ def test_migrate_new(mocker, default_conf, fee, caplog): # Run init to test migration init_db(default_conf['db_url']) - trades = Trade.session.scalars(select(Trade).filter(Trade.id == 1)).all() + trades = Trade.session.scalars(select(Trade)).all() assert len(trades) == 1 trade = trades[0] + assert trade.id == 1 assert trade.fee_open == fee.return_value assert trade.fee_close == fee.return_value assert trade.open_rate_requested is None @@ -281,12 +300,18 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert orders[1].order_id == 'dry_buy_order22' assert orders[1].ft_order_side == 'buy' - assert orders[1].ft_is_open is False + assert orders[1].ft_is_open is True assert orders[2].order_id == 'dry_stop_order_id11X' assert orders[2].ft_order_side == 'stoploss' assert orders[2].ft_is_open is False + orders1 = Order.session.scalars(select(Order)).all() + assert len(orders1) == 5 + order = orders1[4] + assert order.ft_trade_id == 2 + assert order.ft_is_open is False + def test_migrate_too_old(mocker, default_conf, fee, caplog): """ diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 5adaba1d6..396d60c18 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -10,7 +10,8 @@ from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException from freqtrade.persistence import LocalTrade, Order, Trade, init_db from freqtrade.util import dt_now -from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re +from tests.conftest import (create_mock_trades, create_mock_trades_usdt, + create_mock_trades_with_leverage, log_has, log_has_re) spot, margin, futures = TradingMode.SPOT, TradingMode.MARGIN, TradingMode.FUTURES @@ -77,18 +78,28 @@ def test_set_stop_loss_liquidation(fee): assert trade.liquidation_price == 0.11 # Stoploss does not change from liquidation price assert trade.stop_loss == 1.8 + assert trade.stop_loss_pct == -0.2 assert trade.initial_stop_loss == 1.8 # lower stop doesn't move stoploss trade.adjust_stop_loss(1.8, 0.2) assert trade.liquidation_price == 0.11 assert trade.stop_loss == 1.8 + assert trade.stop_loss_pct == -0.2 + assert trade.initial_stop_loss == 1.8 + + # Lower stop with "allow_refresh" does move stoploss + trade.adjust_stop_loss(1.8, 0.22, allow_refresh=True) + assert trade.liquidation_price == 0.11 + assert trade.stop_loss == 1.602 + assert trade.stop_loss_pct == -0.22 assert trade.initial_stop_loss == 1.8 # higher stop does move stoploss trade.adjust_stop_loss(2.1, 0.1) assert trade.liquidation_price == 0.11 assert pytest.approx(trade.stop_loss) == 1.994999 + assert trade.stop_loss_pct == -0.1 assert trade.initial_stop_loss == 1.8 assert trade.stoploss_or_liquidation == trade.stop_loss @@ -130,12 +141,21 @@ def test_set_stop_loss_liquidation(fee): assert trade.liquidation_price == 3.8 # Stoploss does not change from liquidation price assert trade.stop_loss == 2.2 + assert trade.stop_loss_pct == -0.2 assert trade.initial_stop_loss == 2.2 # Stop doesn't move stop higher trade.adjust_stop_loss(2.0, 0.3) assert trade.liquidation_price == 3.8 assert trade.stop_loss == 2.2 + assert trade.stop_loss_pct == -0.2 + assert trade.initial_stop_loss == 2.2 + + # Stop does move stop higher with "allow_refresh" + trade.adjust_stop_loss(2.0, 0.3, allow_refresh=True) + assert trade.liquidation_price == 3.8 + assert trade.stop_loss == 2.3 + assert trade.stop_loss_pct == -0.3 assert trade.initial_stop_loss == 2.2 # Stoploss does move lower @@ -143,6 +163,7 @@ def test_set_stop_loss_liquidation(fee): trade.adjust_stop_loss(1.8, 0.1) assert trade.liquidation_price == 1.5 assert pytest.approx(trade.stop_loss) == 1.89 + assert trade.stop_loss_pct == -0.1 assert trade.initial_stop_loss == 2.2 assert trade.stoploss_or_liquidation == 1.5 @@ -437,15 +458,14 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ leverage=lev, trading_mode=trading_mode ) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None - trade.open_order_id = enter_order['id'] oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', entry_side) trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == open_rate assert trade.close_profit is None assert trade.close_date is None @@ -456,13 +476,12 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_ caplog) caplog.clear() - trade.open_order_id = enter_order['id'] time_machine.move_to("2022-03-31 21:45:05 +00:00") oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side) trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_rate == close_rate assert pytest.approx(trade.close_profit) == profit assert trade.close_date is not None @@ -491,11 +510,10 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, leverage=1.0, ) - trade.open_order_id = 'mocked_market_buy' oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy') trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 2.0 assert trade.close_profit is None assert trade.close_date is None @@ -506,11 +524,10 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee, caplog.clear() trade.is_open = True - trade.open_order_id = 'mocked_market_sell' oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell') trade.orders.append(oobj) trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_rate == 2.2 assert pytest.approx(trade.close_profit) == 0.094513715710723 assert trade.close_date is not None @@ -560,7 +577,6 @@ def test_calc_open_close_trade_price( ) entry_order = limit_order[trade.entry_side] exit_order = limit_order[trade.exit_side] - trade.open_order_id = f'something-{is_short}-{lev}-{exchange}' oobj = Order.parse_from_ccxt_object(entry_order, 'ADA/USDT', trade.entry_side) oobj._trade_live = trade @@ -584,7 +600,9 @@ def test_calc_open_close_trade_price( @pytest.mark.usefixtures("init_persistence") -def test_trade_close(fee): +def test_trade_close(fee, time_machine): + time_machine.move_to("2022-09-01 05:00:00 +00:00", tick=False) + trade = Trade( pair='ADA/USDT', stake_amount=60.0, @@ -593,7 +611,7 @@ def test_trade_close(fee): is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10), + open_date=dt_now() - timedelta(minutes=10), interest_rate=0.0005, exchange='binance', trading_mode=margin, @@ -612,6 +630,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.entry_side, + order_filled_date=trade.open_date, )) trade.orders.append(Order( ft_order_side=trade.exit_side, @@ -626,6 +645,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.exit_side, + order_filled_date=dt_now(), )) assert trade.close_profit is None assert trade.close_date is None @@ -634,14 +654,15 @@ def test_trade_close(fee): assert trade.is_open is False assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None + assert trade.close_date_utc == dt_now() - new_date = datetime(2020, 2, 2, 15, 6, 1), - assert trade.close_date != new_date + new_date = dt_now() + timedelta(minutes=5) + assert trade.close_date_utc != new_date # Close should NOT update close_date if the trade has been closed already assert trade.is_open is False trade.close_date = new_date trade.close(2.2) - assert trade.close_date == new_date + assert trade.close_date_utc == new_date @pytest.mark.usefixtures("init_persistence") @@ -658,7 +679,6 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee): leverage=1.0, ) - trade.open_order_id = 'something' oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) assert trade.calc_close_trade_value(trade.close_rate) == 0.0 @@ -677,7 +697,7 @@ def test_update_open_order(limit_buy_order_usdt): trading_mode=margin ) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None @@ -685,7 +705,7 @@ def test_update_open_order(limit_buy_order_usdt): oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy') trade.update_trade(oobj) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.close_profit is None assert trade.close_date is None @@ -758,7 +778,6 @@ def test_calc_open_trade_value( is_short=is_short, trading_mode=trading_mode ) - trade.open_order_id = 'open_trade' oobj = Order.parse_from_ccxt_object( limit_buy_order_usdt, 'ADA/USDT', 'sell' if is_short else 'buy') trade.update_trade(oobj) # Buy @ 2.0 @@ -813,7 +832,6 @@ def test_calc_close_trade_price( trading_mode=trading_mode, funding_fees=funding_fees ) - trade.open_order_id = 'close_trade' assert round(trade.calc_close_trade_value(rate=close_rate), 8) == result @@ -1132,14 +1150,30 @@ def test_calc_profit( leverage=lev, fee_open=0.0025, fee_close=fee_close, + max_stake_amount=60.0, trading_mode=trading_mode, funding_fees=funding_fees ) - trade.open_order_id = 'something' + + profit_res = trade.calculate_profit(close_rate) + assert pytest.approx(profit_res.profit_abs) == round(profit, 8) + assert pytest.approx(profit_res.profit_ratio) == round(profit_ratio, 8) + val = trade.open_trade_value * (profit_res.profit_ratio) / lev + assert pytest.approx(val) == profit_res.profit_abs + + assert pytest.approx(profit_res.total_profit) == round(profit, 8) + # assert pytest.approx(profit_res.total_profit_ratio) == round(profit_ratio, 8) assert pytest.approx(trade.calc_profit(rate=close_rate)) == round(profit, 8) assert pytest.approx(trade.calc_profit_ratio(rate=close_rate)) == round(profit_ratio, 8) + profit_res2 = trade.calculate_profit(close_rate, trade.amount, trade.open_rate) + assert pytest.approx(profit_res2.profit_abs) == round(profit, 8) + assert pytest.approx(profit_res2.profit_ratio) == round(profit_ratio, 8) + + assert pytest.approx(profit_res2.total_profit) == round(profit, 8) + # assert pytest.approx(profit_res2.total_profit_ratio) == round(profit_ratio, 8) + assert pytest.approx(trade.calc_profit(close_rate, trade.amount, trade.open_rate)) == round(profit, 8) assert pytest.approx(trade.calc_profit_ratio(close_rate, trade.amount, @@ -1315,6 +1349,24 @@ def test_get_open_lev(fee, use_db): Trade.use_db = True +@pytest.mark.parametrize('is_short', [True, False]) +@pytest.mark.parametrize('use_db', [True, False]) +@pytest.mark.usefixtures("init_persistence") +def test_get_open_orders(fee, is_short, use_db): + Trade.use_db = use_db + Trade.reset_trades() + + create_mock_trades_usdt(fee, is_short, use_db) + # Trade.commit() + trade = Trade.get_trades_proxy(pair="XRP/USDT")[0] + # assert trade.id == 3 + assert len(trade.orders) == 2 + assert len(trade.open_orders) == 0 + assert not trade.has_open_orders + + Trade.use_db = True + + @pytest.mark.usefixtures("init_persistence") def test_to_json(fee): @@ -1330,10 +1382,10 @@ def test_to_json(fee): open_rate=0.123, exchange='binance', enter_tag=None, - open_order_id='dry_run_buy_12345', precision_mode=1, amount_precision=8.0, price_precision=7.0, + contract_size=1, ) result = trade.to_json() assert isinstance(result, dict) @@ -1346,7 +1398,6 @@ def test_to_json(fee): 'is_open': None, 'open_date': trade.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': int(trade.open_date.timestamp() * 1000), - 'open_order_id': 'dry_run_buy_12345', 'close_date': None, 'close_timestamp': None, 'open_rate': 0.123, @@ -1400,7 +1451,9 @@ def test_to_json(fee): 'amount_precision': 8.0, 'price_precision': 7.0, 'precision_mode': 1, + 'contract_size': 1, 'orders': [], + 'has_open_orders': False, } # Simulate dry_run entries @@ -1420,6 +1473,7 @@ def test_to_json(fee): precision_mode=2, amount_precision=7.0, price_precision=8.0, + contract_size=1 ) result = trade.to_json() assert isinstance(result, dict) @@ -1468,7 +1522,6 @@ def test_to_json(fee): 'is_open': None, 'max_rate': None, 'min_rate': None, - 'open_order_id': None, 'open_rate_requested': None, 'open_trade_value': 12.33075, 'exit_reason': None, @@ -1486,7 +1539,9 @@ def test_to_json(fee): 'amount_precision': 7.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, 'orders': [], + 'has_open_orders': False, } @@ -1877,11 +1932,15 @@ def test_get_best_pair_lev(fee): @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize('is_short', [True, False]) -def test_get_exit_order_count(fee, is_short): +def test_get_canceled_exit_order_count(fee, is_short): create_mock_trades(fee, is_short=is_short) trade = Trade.get_trades([Trade.pair == 'ETC/BTC']).first() - assert trade.get_exit_order_count() == 1 + # No canceled order. + assert trade.get_canceled_exit_order_count() == 0 + + trade.orders[-1].status = 'canceled' + assert trade.get_canceled_exit_order_count() == 1 @pytest.mark.usefixtures("init_persistence") @@ -2029,7 +2088,6 @@ def test_Trade_object_idem(): 'total_open_trades_stakes', 'get_closed_trades_without_assigned_fees', 'get_open_trades_without_assigned_fees', - 'get_open_order_trades', 'get_trades', 'get_trades_query', 'get_exit_reason_performance', @@ -2639,7 +2697,7 @@ def test_recalc_trade_from_orders_dca(data) -> None: assert len(trade.orders) == idx + 1 if idx < len(data) - 1: assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == result[0] assert trade.open_rate == result[1] assert trade.stake_amount == result[2] @@ -2653,4 +2711,4 @@ def test_recalc_trade_from_orders_dca(data) -> None: assert not trade.is_open trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders diff --git a/tests/persistence/test_trade_fromjson.py b/tests/persistence/test_trade_fromjson.py index 24522e744..24a693c75 100644 --- a/tests/persistence/test_trade_fromjson.py +++ b/tests/persistence/test_trade_fromjson.py @@ -66,6 +66,10 @@ def test_trade_fromjson(): "is_short": false, "trading_mode": "spot", "funding_fees": 0.0, + "amount_precision": 1.0, + "price_precision": 3.0, + "precision_mode": 2, + "contract_size": 1.0, "open_order_id": null, "orders": [ { @@ -180,6 +184,9 @@ def test_trade_fromjson(): assert isinstance(trade.open_date, datetime) assert trade.exit_reason == 'no longer good' assert trade.realized_profit == 2.76315361 + assert trade.precision_mode == 2 + assert trade.amount_precision == 1.0 + assert trade.contract_size == 1.0 assert len(trade.orders) == 5 last_o = trade.orders[-1] diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 369a7f223..d66a47aa6 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -14,7 +14,7 @@ from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade, Trade from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver @@ -616,6 +616,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}], "BTC", "binance", ['ETH/BTC', 'LTC/BTC', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), + # TKN/BTC is removed because it doesn't have enough candles + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": 6, "refresh_period": 86400}], + "BTC", "binance", ['LTC/BTC', 'XRP/BTC', 'ETH/BTC', 'HOT/BTC', 'NEO/BTC']), # ftx data is already in Quote currency, therefore won't require conversion # ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", # "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], @@ -626,23 +630,25 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency whitelist_conf['exchange']['name'] = exchange + # Ensure we have 6 candles + ohlcv_history_long = pd.concat([ohlcv_history, ohlcv_history]) - ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola = ohlcv_history_long.copy() ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 # create candles for medium overall volume with last candle high volume - ohlcv_history_medium_volume = ohlcv_history.copy() + ohlcv_history_medium_volume = ohlcv_history_long.copy() ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5 # create candles for high volume with all candles high volume, but very low price. - ohlcv_history_high_volume = ohlcv_history.copy() + ohlcv_history_high_volume = ohlcv_history_long.copy() ohlcv_history_high_volume['volume'] = 10 ohlcv_history_high_volume['low'] = ohlcv_history_high_volume.loc[:, 'low'] * 0.01 ohlcv_history_high_volume['high'] = ohlcv_history_high_volume.loc[:, 'high'] * 0.01 ohlcv_history_high_volume['close'] = ohlcv_history_high_volume.loc[:, 'close'] * 0.01 ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history_long, ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history_medium_volume, ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola, @@ -1370,7 +1376,12 @@ def test_expand_pairlist(wildcardlist, pairs, expected): (['BTC/USD'], ['BTC/USD', 'BTC/USDT'], ['BTC/USD']), - + (['BTC/USDT:USDT'], + ['BTC/USDT:USDT', 'BTC/USDT'], + ['BTC/USDT:USDT']), + (['BB_BTC/USDT', 'CC_BTC/USDT', 'AA_ETH/USDT', 'XRP/USDT', 'ETH/USDT', 'XX_BTC/USDT'], + ['BTC/USDT', 'ETH/USDT'], + ['XRP/USDT', 'ETH/USDT']), ]) def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected): if expected is None: @@ -1452,3 +1463,53 @@ def test_ProducerPairlist(mocker, whitelist_conf, markets): pm.refresh_pairlist() assert len(pm.whitelist) == 4 assert pm.whitelist == ['TKN/BTC'] + pairs + + +@pytest.mark.usefixtures("init_persistence") +def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: + default_conf_usdt['exchange']['pair_whitelist'].extend(['ADA/USDT', 'XRP/USDT', 'ETC/USDT']) + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "FullTradesFilter"} + ] + default_conf_usdt['max_open_trades'] = -1 + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, default_conf_usdt) + pm = PairListManager(exchange, default_conf_usdt) + pm.refresh_pairlist() + + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: + create_mock_trades_usdt(fee) + pm.refresh_pairlist() + + # Unlimited max open trades, so no change to whitelist + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + # Set max_open_trades to 4, the filter should empty the whitelist + default_conf_usdt['max_open_trades'] = 4 + pm.refresh_pairlist() + assert pm.whitelist == [] + assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) + + list_trades = LocalTrade.get_open_trades() + assert len(list_trades) == 4 + + # Move to 1 hour later, close a trade, so original sorting is restored. + t.move_to("2021-09-01 07:00:00 +00:00") + list_trades[2].close(12) + Trade.commit() + + # open trades count below max_open_trades, whitelist restored + list_trades = LocalTrade.get_open_trades() + assert len(list_trades) == 3 + pm.refresh_pairlist() + assert pm.whitelist == ['ETH/USDT', 'XRP/USDT', 'NEO/USDT', 'TKN/USDT'] + + # Set max_open_trades to 3, the filter should empty the whitelist + default_conf_usdt['max_open_trades'] = 3 + pm.refresh_pairlist() + assert pm.whitelist == [] + assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index d97222adc..b8eb51a91 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -42,7 +42,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'strategy': ANY, 'enter_tag': ANY, 'timeframe': 5, - 'open_order_id': ANY, 'close_date': None, 'close_timestamp': None, 'open_rate': 1.098e-05, @@ -75,7 +74,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_current_dist_pct': -10.01, 'stoploss_entry_dist': -0.00010402, 'stoploss_entry_dist_ratio': -0.10376381, - 'open_order': None, + 'open_orders': '', 'realized_profit': 0.0, 'realized_profit_ratio': None, 'total_profit_abs': -4.09e-06, @@ -91,6 +90,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'amount_precision': 8.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, + 'has_open_orders': False, 'orders': [{ 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, 'cost': 0.0009999999999054, 'filled': 91.07468123, 'ft_order_side': 'buy', @@ -128,7 +129,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_pct': 0.0, 'profit_abs': 0.0, 'total_profit_abs': 0.0, - 'open_order': '(limit buy rem=91.07468123)', + 'open_orders': '(limit buy rem=91.07468123)', + 'has_open_orders': True, }) response_unfilled['orders'][0].update({ 'is_open': True, @@ -146,7 +148,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: results = rpc._rpc_trade_status() # Reuse above object, only remaining changed. response_unfilled['orders'][0].update({ - 'remaining': None + 'remaining': None, }) assert results[0] == response_unfilled @@ -164,7 +166,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: response = deepcopy(gen_response) response.update({ 'max_stake_amount': 0.001, - 'total_profit_ratio': pytest.approx(-0.00409), + 'total_profit_ratio': pytest.approx(-0.00409153), + 'has_open_orders': False, }) assert results[0] == response @@ -261,7 +264,11 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, markets, mocker) -> None: +def test__rpc_timeunit_profit( + default_conf_usdt, ticker, fee, markets, mocker, time_machine) -> None: + + time_machine.move_to("2023-09-05 10:00:00 +00:00", tick=False) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( EXMS, @@ -779,7 +786,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: 'amount': amount, 'remaining': amount, 'filled': 0.0, - 'id': trade.orders[0].order_id, + 'id': trade.orders[-1].order_id, } ) cancel_order_3 = mocker.patch( @@ -791,7 +798,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: 'amount': amount, 'remaining': amount, 'filled': 0.0, - 'id': trade.orders[0].order_id, + 'id': trade.orders[-1].order_id, } ) msg = rpc._rpc_force_exit('3') @@ -800,7 +807,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert cancel_order_3.call_count == 1 assert cancel_order_mock.call_count == 0 - trade = Trade.session.scalars(select(Trade).filter(Trade.id == '2')).first() + trade = Trade.session.scalars(select(Trade).filter(Trade.id == '4')).first() amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it mocker.patch( @@ -829,7 +836,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None: assert msg == {'result': 'Created exit order for trade 4.'} assert cancel_order_4.call_count == 1 assert cancel_order_mock.call_count == 0 - assert trade.amount == amount + assert pytest.approx(trade.amount) == amount def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None: @@ -1097,7 +1104,8 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) assert trade.stake_amount == 0.05 assert trade.buy_tag == 'force_entry' - assert trade.open_order_id == 'mocked_limit_buy' + + assert trade.open_orders_ids[-1] == 'mocked_limit_buy' freqtradebot.strategy.position_adjustment_enable = True with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index a85d15365..4bbd07f18 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -617,6 +617,47 @@ def test_api_daily(botclient, mocker, ticker, fee, markets): assert rc.json()['data'][0]['date'] == str(datetime.now(timezone.utc).date()) +def test_api_weekly(botclient, mocker, ticker, fee, markets, time_machine): + ftbot, client = botclient + patch_get_signal(ftbot) + mocker.patch.multiple( + EXMS, + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + time_machine.move_to("2023-03-31 21:45:05 +00:00") + rc = client_get(client, f"{BASE_URI}/weekly") + assert_response(rc) + assert len(rc.json()['data']) == 4 + assert rc.json()['stake_currency'] == 'BTC' + assert rc.json()['fiat_display_currency'] == 'USD' + # Moved to monday + assert rc.json()['data'][0]['date'] == '2023-03-27' + assert rc.json()['data'][1]['date'] == '2023-03-20' + + +def test_api_monthly(botclient, mocker, ticker, fee, markets, time_machine): + ftbot, client = botclient + patch_get_signal(ftbot) + mocker.patch.multiple( + EXMS, + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + time_machine.move_to("2023-03-31 21:45:05 +00:00") + rc = client_get(client, f"{BASE_URI}/monthly") + assert_response(rc) + assert len(rc.json()['data']) == 3 + assert rc.json()['stake_currency'] == 'BTC' + assert rc.json()['fiat_display_currency'] == 'USD' + assert rc.json()['data'][0]['date'] == '2023-03-01' + assert rc.json()['data'][1]['date'] == '2023-02-01' + + @pytest.mark.parametrize('is_short', [True, False]) def test_api_trades(botclient, mocker, fee, markets, is_short): ftbot, client = botclient @@ -936,6 +977,10 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected) 'expectancy_ratio': expected['expectancy_ratio'], 'max_drawdown': ANY, 'max_drawdown_abs': ANY, + 'max_drawdown_start': ANY, + 'max_drawdown_start_timestamp': ANY, + 'max_drawdown_end': ANY, + 'max_drawdown_end_timestamp': ANY, 'trading_volume': expected['trading_volume'], 'bot_start_timestamp': 0, 'bot_start_date': '', @@ -981,12 +1026,11 @@ def test_api_performance(botclient, fee): exchange='binance', stake_amount=1, open_rate=0.245441, - open_order_id="123456", is_open=False, fee_close=fee.return_value, fee_open=fee.return_value, close_rate=0.265441, - + leverage=1.0, ) trade.close_profit = trade.calc_profit_ratio(trade.close_rate) trade.close_profit_abs = trade.calc_profit(trade.close_rate) @@ -998,11 +1042,11 @@ def test_api_performance(botclient, fee): stake_amount=1, exchange='binance', open_rate=0.412, - open_order_id="123456", is_open=False, fee_close=fee.return_value, fee_open=fee.return_value, - close_rate=0.391 + close_rate=0.391, + leverage=1.0, ) trade.close_profit = trade.calc_profit_ratio(trade.close_rate) trade.close_profit_abs = trade.calc_profit(trade.close_rate) @@ -1020,11 +1064,11 @@ def test_api_performance(botclient, fee): @pytest.mark.parametrize( - 'is_short,current_rate,open_order_id,open_trade_value', - [(True, 1.098e-05, 'dry_run_buy_short_12345', 15.0911775), - (False, 1.099e-05, 'dry_run_buy_long_12345', 15.1668225)]) + 'is_short,current_rate,open_trade_value', + [(True, 1.098e-05, 15.0911775), + (False, 1.099e-05, 15.1668225)]) def test_api_status(botclient, mocker, ticker, fee, markets, is_short, - current_rate, open_order_id, open_trade_value): + current_rate, open_trade_value): ftbot, client = botclient patch_get_signal(ftbot) mocker.patch.multiple( @@ -1065,7 +1109,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'current_rate': current_rate, 'open_date': ANY, 'open_timestamp': ANY, - 'open_order': None, 'open_rate': 0.123, 'pair': 'ETH/BTC', 'base_currency': 'ETH', @@ -1098,7 +1141,6 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, "is_short": is_short, 'max_rate': ANY, 'min_rate': ANY, - 'open_order_id': open_order_id, 'open_rate_requested': ANY, 'open_trade_value': open_trade_value, 'exit_reason': None, @@ -1116,6 +1158,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'price_precision': None, 'precision_mode': None, 'orders': [ANY], + 'has_open_orders': True, } mocker.patch(f'{EXMS}.get_rate', @@ -1245,7 +1288,6 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): exchange='binance', stake_amount=1, open_rate=0.245441, - open_order_id="123456", open_date=datetime.now(timezone.utc), is_open=False, is_short=False, @@ -1306,7 +1348,6 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'is_short': False, 'max_rate': None, 'min_rate': None, - 'open_order_id': '123456', 'open_rate_requested': None, 'open_trade_value': 0.24605460, 'exit_reason': None, @@ -1323,6 +1364,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): 'amount_precision': None, 'price_precision': None, 'precision_mode': None, + 'has_open_orders': False, 'orders': [], } @@ -1594,7 +1636,8 @@ def test_api_strategies(botclient, tmpdir): 'freqai_test_classifier', 'freqai_test_multimodel_classifier_strat', 'freqai_test_multimodel_strat', - 'freqai_test_strat' + 'freqai_test_strat', + 'strategy_test_v3_recursive_issue' ]} diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 899bef20e..41c24cc45 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -347,8 +347,8 @@ async def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> msg = msg_mock.call_args_list[3][0][0] assert re.search(r'Number of Entries.*2', msg) assert re.search(r'Number of Exits.*1', msg) - assert re.search(r'Average Entry Price', msg) - assert re.search(r'Order filled', msg) + assert re.search(r'from 1st entry rate', msg) + assert re.search(r'Order Filled', msg) assert re.search(r'Close Date:', msg) is None assert re.search(r'Close Profit:', msg) is None diff --git a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py index 872984156..607ff6e1e 100644 --- a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py +++ b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py @@ -6,6 +6,8 @@ from typing import Optional from pandas import DataFrame from strategy_test_v3 import StrategyTestV3 +from freqtrade.persistence import Trade + class StrategyTestV3CustomEntryPrice(StrategyTestV3): """ @@ -31,7 +33,8 @@ class StrategyTestV3CustomEntryPrice(StrategyTestV3): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return self.new_entry_price diff --git a/tests/strategy/strats/strategy_test_v3_recursive_issue.py b/tests/strategy/strats/strategy_test_v3_recursive_issue.py new file mode 100644 index 000000000..b3074113d --- /dev/null +++ b/tests/strategy/strats/strategy_test_v3_recursive_issue.py @@ -0,0 +1,46 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +import talib.abstract as ta +from pandas import DataFrame + +from freqtrade.strategy import IStrategy +from freqtrade.strategy.parameters import CategoricalParameter + + +class strategy_test_v3_recursive_issue(IStrategy): + INTERFACE_VERSION = 3 + + # Minimal ROI designed for the strategy + minimal_roi = { + "0": 0.04 + } + + # Optimal stoploss designed for the strategy + stoploss = -0.10 + + # Optimal timeframe for the strategy + timeframe = '5m' + scenario = CategoricalParameter(['no_bias', 'bias1', 'bias2'], default='bias1', space="buy") + + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 100 + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # bias is introduced here + if self.scenario.value == 'no_bias': + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + else: + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50) + + if self.scenario.value == 'bias2': + # Has both bias1 and bias2 + dataframe['rsi_lookahead'] = ta.RSI(dataframe, timeperiod=50).shift(-1) + + return dataframe + + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe + + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + return dataframe diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index b5b07e0cd..afe7fc97a 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -52,4 +52,5 @@ def test_strategy_test_v3(dataframe_1m, fee, is_short, side): side=side) is True assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(), - current_rate=20_000, current_profit=0.05) == strategy.stoploss + current_rate=20_000, current_profit=0.05, after_fill=False + ) == strategy.stoploss diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 8a609cf30..e8abcb362 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -503,6 +503,7 @@ def test_custom_exit(default_conf, fee, caplog) -> None: fee_close=fee.return_value, exchange='binance', open_rate=1, + leverage=1.0, ) now = dt_now() @@ -552,6 +553,7 @@ def test_should_sell(default_conf, fee) -> None: fee_close=fee.return_value, exchange='binance', open_rate=1, + leverage=1.0, ) now = dt_now() res = strategy.should_exit(trade, 1, now, diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index a55580780..535b3dbd6 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -96,6 +96,32 @@ def test_merge_informative_pair_lower(): merge_informative_pair(data, informative, '1h', '15m', ffill=True) +def test_merge_informative_pair_empty(): + data = generate_test_data('1h', 40) + informative = pd.DataFrame(columns=data.columns) + + result = merge_informative_pair(data, informative, '1h', '2h', ffill=True) + assert result['date'].equals(data['date']) + + assert list(result.columns) == [ + 'date', + 'open', + 'high', + 'low', + 'close', + 'volume', + 'date_2h', + 'open_2h', + 'high_2h', + 'low_2h', + 'close_2h', + 'volume_2h' + ] + # We merge an empty dataframe, so all values should be NaN + for col in ['date_2h', 'open_2h', 'high_2h', 'low_2h', 'close_2h', 'volume_2h']: + assert result[col].isnull().all() + + def test_merge_informative_pair_suffix(): data = generate_test_data('15m', 20) informative = generate_test_data('1h', 20) @@ -110,6 +136,21 @@ def test_merge_informative_pair_suffix(): assert 'open_suf' in result.columns assert 'open_1h' not in result.columns + assert list(result.columns) == [ + 'date', + 'open', + 'high', + 'low', + 'close', + 'volume', + 'date_suf', + 'open_suf', + 'high_suf', + 'low_suf', + 'close_suf', + 'volume_suf' + ] + def test_merge_informative_pair_suffix_append_timeframe(): data = generate_test_data('15m', 20) @@ -211,15 +252,18 @@ def test_stoploss_from_absolute(): assert pytest.approx(stoploss_from_absolute(110, 100)) == 0 assert pytest.approx(stoploss_from_absolute(100, 0)) == 1 assert pytest.approx(stoploss_from_absolute(0, 100)) == 1 + assert pytest.approx(stoploss_from_absolute(0, 100, False, leverage=5)) == 5 assert pytest.approx(stoploss_from_absolute(90, 100, True)) == 0 assert pytest.approx(stoploss_from_absolute(100, 100, True)) == 0 assert pytest.approx(stoploss_from_absolute(110, 100, True)) == -(1 - (110 / 100)) assert pytest.approx(stoploss_from_absolute(110, 100, True)) == 0.1 assert pytest.approx(stoploss_from_absolute(105, 100, True)) == 0.05 + assert pytest.approx(stoploss_from_absolute(105, 100, True, 5)) == 0.05 * 5 assert pytest.approx(stoploss_from_absolute(100, 0, True)) == 1 assert pytest.approx(stoploss_from_absolute(0, 100, True)) == 0 - assert pytest.approx(stoploss_from_absolute(100, 1, True)) == 1 + assert pytest.approx(stoploss_from_absolute(100, 1, is_short=True)) == 1 + assert pytest.approx(stoploss_from_absolute(100, 1, is_short=True, leverage=5)) == 5 @pytest.mark.parametrize('trading_mode', ['futures', 'spot']) diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 4cdb35936..a31408b5c 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver._search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 12 + assert len(strategies) == 13 assert isinstance(strategies[0], dict) @@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver._search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 13 + assert len(strategies) == 14 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 12 + assert len([x for x in strategies if x['class'] is not None]) == 13 assert len([x for x in strategies if x['class'] is None]) == 1 diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 088c9ee5e..9791a92ff 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -872,7 +872,8 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade.is_short = is_short assert trade assert trade.is_open is True - assert trade.open_order_id == '22' + assert trade.has_open_orders + assert '22' in trade.open_orders_ids # Test calling with price open_order['id'] = '33' @@ -898,7 +899,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade = Trade.session.scalars(select(Trade)).all()[2] trade.is_short = is_short assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 10 assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) assert pytest.approx(trade.liquidation_price) == liq_price @@ -916,7 +917,7 @@ def test_execute_entry(mocker, default_conf_usdt, fee, limit_order, trade = Trade.session.scalars(select(Trade)).all()[3] trade.is_short = is_short assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 0.5 assert trade.stake_amount == round(order['average'] * order['filled'] / leverage, 8) @@ -1118,7 +1119,6 @@ def test_add_stoploss_on_exchange(mocker, default_conf_usdt, limit_order, is_sho freqtrade.enter_positions() trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short - trade.open_order_id = None trade.stoploss_order_id = None trade.is_open = True trades = [trade] @@ -1163,7 +1163,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1174,7 +1173,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should do nothing and return false stop_order_dict.update({'id': "102"}) trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "102" trade.orders.append( Order( @@ -1198,7 +1196,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ # should set a stoploss immediately and return False caplog.clear() trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "102" canceled_stoploss_order = MagicMock(return_value={'id': '103_1', 'status': 'canceled'}) @@ -1223,7 +1220,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "104" trade.orders.append(Order( ft_order_side='stoploss', @@ -1351,7 +1347,6 @@ def test_handle_stoploss_on_exchange_partial( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1410,7 +1405,6 @@ def test_handle_stoploss_on_exchange_partial_cancel_here( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1468,8 +1462,8 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, ) @@ -1485,7 +1479,6 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, trade = Trade.session.scalars(select(Trade)).first() assert trade.is_short == is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "100" trade.orders.append( Order( @@ -1663,7 +1656,6 @@ def test_handle_stoploss_on_exchange_trailing( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() - timedelta(minutes=20) trade.orders.append( @@ -1793,7 +1785,6 @@ def test_handle_stoploss_on_exchange_trailing_error( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = "abcd" trade.stop_loss = 0.2 trade.stoploss_last_update = (dt_now() - timedelta(minutes=601)).replace(tzinfo=None) @@ -1873,8 +1864,8 @@ def test_handle_stoploss_on_exchange_custom_stop( 'last': 1.9 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, ) @@ -1907,7 +1898,6 @@ def test_handle_stoploss_on_exchange_custom_stop( trade = Trade.session.scalars(select(Trade)).first() trade.is_short = is_short trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() - timedelta(minutes=601) trade.orders.append( @@ -2045,7 +2035,6 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde freqtrade.enter_positions() trade = Trade.session.scalars(select(Trade)).first() trade.is_open = True - trade.open_order_id = None trade.stoploss_order_id = '100' trade.stoploss_last_update = dt_now() trade.orders.append( @@ -2152,7 +2141,6 @@ def test_exit_positions(mocker, default_conf_usdt, limit_order, is_short, caplog order_id = '123' trade = Trade( - open_order_id=order_id, pair='ETH/USDT', fee_open=0.001, fee_close=0.001, @@ -2198,7 +2186,6 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog order_id = '123' trade = Trade( - open_order_id=order_id, pair='ETH/USDT', fee_open=0.001, fee_close=0.001, @@ -2217,9 +2204,9 @@ def test_exit_positions_exception(mocker, default_conf_usdt, limit_order, caplog ft_amount=trade.amount, ft_price=trade.open_rate, order_id=order_id, + ft_is_open=False, )) - trade.open_order_id = None Trade.session.add(trade) Trade.commit() freqtrade.wallets.update() @@ -2248,7 +2235,6 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca order_id = order['id'] trade = Trade( - open_order_id=order_id, fee_open=0.001, fee_close=0.001, open_rate=0.01, @@ -2272,19 +2258,17 @@ def test_update_trade_state(mocker, default_conf_usdt, limit_order, is_short, ca # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) caplog.clear() - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == order['amount'] - trade.open_order_id = order_id mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=0.01) assert trade.amount == 30.0 # test amount modified by fee-logic freqtrade.update_trade_state(trade, order_id) assert trade.amount == 29.99 - assert trade.open_order_id is None + assert not trade.has_open_orders trade.is_open = True - trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution freqtrade.update_trade_state(trade, order_id) @@ -2328,7 +2312,6 @@ def test_update_trade_state_withorderdict( open_date=dt_now(), fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id=order_id, is_open=True, leverage=1, is_short=is_short, @@ -2361,15 +2344,15 @@ def test_update_trade_state_exception(mocker, default_conf_usdt, is_short, limit # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '123' trade.amount = 123 + open_order_id = '123' # Test raise of OperationalException exception mocker.patch( 'freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=DependencyException() ) - freqtrade.update_trade_state(trade, trade.open_order_id) + freqtrade.update_trade_state(trade, open_order_id) assert log_has('Could not update trade amount: ', caplog) @@ -2379,13 +2362,13 @@ def test_update_trade_state_orderexception(mocker, default_conf_usdt, caplog) -> # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '123' + open_order_id = '123' # Test raise of OperationalException exception grm_mock = mocker.patch("freqtrade.freqtradebot.FreqtradeBot.get_real_amount", MagicMock()) - freqtrade.update_trade_state(trade, trade.open_order_id) + freqtrade.update_trade_state(trade, open_order_id) assert grm_mock.call_count == 0 - assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) + assert log_has(f'Unable to fetch order {open_order_id}: ', caplog) @pytest.mark.parametrize("is_short", [False, True]) @@ -2413,7 +2396,6 @@ def test_update_trade_state_sell( fee_open=0.0025, fee_close=0.0025, open_date=dt_now(), - open_order_id=open_order['id'], is_open=True, interest_rate=0.0005, leverage=1, @@ -2425,7 +2407,7 @@ def test_update_trade_state_sell( order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', exit_side(is_short)) trade.orders.append(order) assert order.status == 'open' - freqtrade.update_trade_state(trade, trade.open_order_id, l_order) + freqtrade.update_trade_state(trade, trade.open_orders_ids[-1], l_order) assert trade.amount == l_order['amount'] # Wallet needs to be updated after closing a limit-sell order to reenable buying assert wallet_mock.call_count == 1 @@ -2475,7 +2457,7 @@ def test_handle_trade( patch_get_signal(freqtrade, enter_long=False, exit_short=is_short, exit_long=not is_short, exit_tag='sell_signal1') assert freqtrade.handle_trade(trade) is True - assert trade.open_order_id == exit_order['id'] + assert trade.open_orders_ids[-1] == exit_order['id'] # Simulate fulfilled LIMIT_SELL order for trade trade.orders[-1].ft_is_open = False @@ -2706,7 +2688,7 @@ def test_manage_open_orders_entry_usercustom( ) -> None: old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id + old_order['id'] = open_trade.open_orders_ids[0] default_conf_usdt["unfilledtimeout"] = {"entry": 1400, "exit": 30} @@ -2741,7 +2723,11 @@ def test_manage_open_orders_entry_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2750,7 +2736,11 @@ def test_manage_open_orders_entry_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2761,7 +2751,11 @@ def test_manage_open_orders_entry_usercustom( assert cancel_order_wr_mock.call_count == 1 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 0 assert freqtrade.strategy.check_entry_timeout.call_count == 1 @@ -2774,12 +2768,12 @@ def test_manage_open_orders_entry( ) -> None: old_order = limit_sell_order_old if is_short else limit_buy_order_old rpc_mock = patch_RPCManager(mocker) - open_trade.open_order_id = old_order['id'] + order = Order.parse_from_ccxt_object(old_order, 'mocked', 'buy') open_trade.orders[0] = order - limit_buy_cancel = deepcopy(old_order) - limit_buy_cancel['status'] = 'canceled' - cancel_order_mock = MagicMock(return_value=limit_buy_cancel) + limit_entry_cancel = deepcopy(old_order) + limit_entry_cancel['status'] = 'canceled' + cancel_order_mock = MagicMock(return_value=limit_entry_cancel) patch_exchange(mocker) mocker.patch.multiple( EXMS, @@ -2801,7 +2795,11 @@ def test_manage_open_orders_entry( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() nb_trades = len(trades) assert nb_trades == 0 # Custom user entry-timeout is never called @@ -2817,10 +2815,10 @@ def test_adjust_entry_cancel( ) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id - limit_buy_cancel = deepcopy(old_order) - limit_buy_cancel['status'] = 'canceled' - cancel_order_mock = MagicMock(return_value=limit_buy_cancel) + old_order['id'] = open_trade.open_orders[0].order_id + limit_entry_cancel = deepcopy(old_order) + limit_entry_cancel['status'] = 'canceled' + cancel_order_mock = MagicMock(return_value=limit_entry_cancel) mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, @@ -2840,7 +2838,10 @@ def test_adjust_entry_cancel( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=None) freqtrade.manage_open_orders() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_trade_id == Trade.id) + ).all() + assert len(trades) == 0 assert len(Order.session.scalars(select(Order)).all()) == 0 assert log_has_re( @@ -2852,6 +2853,52 @@ def test_adjust_entry_cancel( assert freqtrade.strategy.adjust_entry_price.call_count == 1 +@pytest.mark.parametrize("is_short", [False, True]) +def test_adjust_entry_replace_fail( + default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, + limit_sell_order_old, fee, mocker, caplog, is_short +) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + old_order = limit_sell_order_old if is_short else limit_buy_order_old + old_order['id'] = open_trade.open_orders[0].order_id + limit_entry_cancel = deepcopy(old_order) + limit_entry_cancel['status'] = 'open' + cancel_order_mock = MagicMock(return_value=limit_entry_cancel) + fetch_order_mock = MagicMock(return_value=old_order) + mocker.patch.multiple( + EXMS, + fetch_ticker=ticker_usdt, + fetch_order=fetch_order_mock, + cancel_order_with_result=cancel_order_mock, + get_fee=fee + ) + mocker.patch('freqtrade.freqtradebot.sleep') + + open_trade.is_short = is_short + Trade.session.add(open_trade) + Trade.commit() + + # Timeout to not interfere + freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) + + # Attempt replace order - which fails + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=12234) + freqtrade.manage_open_orders() + trades = Trade.session.scalars( + select(Trade) + .where(Order.ft_trade_id == Trade.id) + ).all() + + assert len(trades) == 0 + assert len(Order.session.scalars(select(Order)).all()) == 0 + assert fetch_order_mock.call_count == 4 + assert log_has_re( + r"Could not cancel order.*, therefore not replacing\.", caplog) + + # Entry adjustment is called + assert freqtrade.strategy.adjust_entry_price.call_count == 1 + + @pytest.mark.parametrize("is_short", [False, True]) def test_adjust_entry_maintain_replace( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, @@ -2859,16 +2906,17 @@ def test_adjust_entry_maintain_replace( ) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old - old_order['id'] = open_trade.open_order_id - limit_buy_cancel = deepcopy(old_order) - limit_buy_cancel['status'] = 'canceled' - cancel_order_mock = MagicMock(return_value=limit_buy_cancel) + old_order['id'] = open_trade.open_orders_ids[0] + limit_entry_cancel = deepcopy(old_order) + limit_entry_cancel['status'] = 'canceled' + cancel_order_mock = MagicMock(return_value=limit_entry_cancel) mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, fetch_order=MagicMock(return_value=old_order), cancel_order_with_result=cancel_order_mock, - get_fee=fee + get_fee=fee, + _dry_is_price_crossed=MagicMock(return_value=False), ) open_trade.is_short = is_short @@ -2882,7 +2930,10 @@ def test_adjust_entry_maintain_replace( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=old_order['price']) freqtrade.manage_open_orders() trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 assert len(Order.get_open_orders()) == 1 # Entry adjustment is called @@ -2891,9 +2942,16 @@ def test_adjust_entry_maintain_replace( # Check that order is replaced freqtrade.get_valid_enter_price_and_stake = MagicMock(return_value={100, 10, 1}) freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) + freqtrade.manage_open_orders() + + assert freqtrade.strategy.adjust_entry_price.call_count == 1 + trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 nb_all_orders = len(Order.session.scalars(select(Order)).all()) assert nb_all_orders == 2 @@ -2917,6 +2975,8 @@ def test_check_handle_cancelled_buy( cancel_order_mock = MagicMock() patch_exchange(mocker) old_order.update({"status": "canceled", 'filled': 0.0}) + old_order['side'] = 'buy' if is_short else 'sell' + old_order['id'] = open_trade.open_orders[0].order_id mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, @@ -2925,7 +2985,6 @@ def test_check_handle_cancelled_buy( get_fee=fee ) freqtrade = FreqtradeBot(default_conf_usdt) - open_trade.orders = [] open_trade.is_short = is_short Trade.session.add(open_trade) Trade.commit() @@ -2935,10 +2994,13 @@ def test_check_handle_cancelled_buy( assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 2 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 0 - assert log_has_re( - f"{'Sell' if is_short else 'Buy'} order cancelled on exchange for Trade.*", caplog) + exit_name = 'Buy' if is_short else 'Sell' + assert log_has_re(f"{exit_name} order cancelled on exchange for Trade.*", caplog) @pytest.mark.parametrize("is_short", [False, True]) @@ -2966,10 +3028,7 @@ def test_manage_open_orders_buy_exception( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() - nb_trades = len(trades) - assert nb_trades == 1 + assert len(open_trade.open_orders) == 1 @pytest.mark.parametrize("is_short", [False, True]) @@ -2978,7 +3037,9 @@ def test_manage_open_orders_exit_usercustom( is_short, open_trade_usdt, caplog ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} - open_trade_usdt.open_order_id = limit_sell_order_old['id'] + limit_sell_order_old['amount'] = open_trade_usdt.amount + limit_sell_order_old['remaining'] = open_trade_usdt.amount + if is_short: limit_sell_order_old['side'] = 'buy' open_trade_usdt.is_short = is_short @@ -3035,14 +3096,11 @@ def test_manage_open_orders_exit_usercustom( assert rpc_mock.call_count == 2 assert freqtrade.strategy.check_exit_timeout.call_count == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 0 - trade = Trade.session.scalars(select(Trade)).first() - # cancelling didn't succeed - order-id remains open. - assert trade.open_order_id is not None # 2nd canceled trade - Fail execute exit caplog.clear() - open_trade_usdt.open_order_id = limit_sell_order_old['id'] - mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) + + mocker.patch('freqtrade.persistence.Trade.get_canceled_exit_order_count', return_value=1) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', side_effect=DependencyException) freqtrade.manage_open_orders() @@ -3051,7 +3109,6 @@ def test_manage_open_orders_exit_usercustom( et_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit') caplog.clear() # 2nd canceled trade ... - open_trade_usdt.open_order_id = limit_sell_order_old['id'] # If cancelling fails - no emergency exit! with patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit', return_value=False): @@ -3069,7 +3126,7 @@ def test_manage_open_orders_exit( ) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() - limit_sell_order_old['id'] = open_trade_usdt.open_order_id + limit_sell_order_old['id'] = '123456789_exit' limit_sell_order_old['side'] = 'buy' if is_short else 'sell' patch_exchange(mocker) mocker.patch.multiple( @@ -3111,7 +3168,7 @@ def test_check_handle_cancelled_exit( cancel_order_mock = MagicMock() limit_sell_order_old.update({"status": "canceled", 'filled': 0.0}) limit_sell_order_old['side'] = 'buy' if is_short else 'sell' - limit_sell_order_old['id'] = open_trade_usdt.open_order_id + limit_sell_order_old['id'] = open_trade_usdt.open_orders[0].order_id patch_exchange(mocker) mocker.patch.multiple( @@ -3148,7 +3205,8 @@ def test_manage_open_orders_partial( open_trade.is_short = is_short open_trade.leverage = leverage open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' - limit_buy_order_old_partial['id'] = open_trade.open_order_id + + limit_buy_order_old_partial['id'] = open_trade.orders[0].order_id limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_canceled = deepcopy(limit_buy_order_old_partial) limit_buy_canceled['status'] = 'canceled' @@ -3172,11 +3230,13 @@ def test_manage_open_orders_partial( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + ).all() assert len(trades) == 1 assert trades[0].amount == 23.0 assert trades[0].stake_amount == open_trade.open_rate * trades[0].amount / leverage assert trades[0].stake_amount != prior_stake + assert not trades[0].has_open_orders @pytest.mark.parametrize("is_short", [False, True]) @@ -3188,8 +3248,8 @@ def test_manage_open_orders_partial_fee( open_trade.is_short = is_short open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) - limit_buy_order_old_partial['id'] = open_trade.open_order_id - limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id + limit_buy_order_old_partial['id'] = open_trade.orders[0].order_id + limit_buy_order_old_partial_canceled['id'] = open_trade.open_orders_ids[0] limit_buy_order_old_partial['side'] = 'sell' if is_short else 'buy' limit_buy_order_old_partial_canceled['side'] = 'sell' if is_short else 'buy' @@ -3220,12 +3280,14 @@ def test_manage_open_orders_partial_fee( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + .where(Order.ft_trade_id == Trade.id) + ).all() assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - limit_buy_order_old_partial['remaining']) - 0.023 - assert trades[0].open_order_id is None + assert not trades[0].has_open_orders assert trades[0].fee_updated(open_trade.entry_side) assert pytest.approx(trades[0].fee_open) == 0.001 @@ -3239,8 +3301,8 @@ def test_manage_open_orders_partial_except( open_trade.is_short = is_short open_trade.orders[0].ft_order_side = 'sell' if is_short else 'buy' rpc_mock = patch_RPCManager(mocker) - limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id - limit_buy_order_old_partial['id'] = open_trade.open_order_id + limit_buy_order_old_partial_canceled['id'] = open_trade.open_orders_ids[0] + limit_buy_order_old_partial['id'] = open_trade.open_orders_ids[0] if is_short: limit_buy_order_old_partial['side'] = 'sell' cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) @@ -3271,13 +3333,14 @@ def test_manage_open_orders_partial_except( assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 3 trades = Trade.session.scalars( - select(Trade).filter(Trade.open_order_id.is_(open_trade.open_order_id))).all() + select(Trade) + ).all() assert len(trades) == 1 # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - limit_buy_order_old_partial['remaining']) - assert trades[0].open_order_id is None + assert not trades[0].has_open_orders assert trades[0].fee_open == fee() @@ -3317,12 +3380,12 @@ def test_manage_open_orders_exception(default_conf_usdt, ticker_usdt, open_trade def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_short, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - l_order = limit_order[entry_side(is_short)] - cancel_buy_order = deepcopy(limit_order[entry_side(is_short)]) - cancel_buy_order['status'] = 'canceled' - del cancel_buy_order['filled'] + l_order = deepcopy(limit_order[entry_side(is_short)]) + cancel_entry_order = deepcopy(limit_order[entry_side(is_short)]) + cancel_entry_order['status'] = 'canceled' + del cancel_entry_order['filled'] - cancel_order_mock = MagicMock(return_value=cancel_buy_order) + cancel_order_mock = MagicMock(return_value=cancel_entry_order) mocker.patch(f'{EXMS}.cancel_order_with_result', cancel_order_mock) freqtrade = FreqtradeBot(default_conf_usdt) @@ -3335,32 +3398,47 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_ l_order['filled'] = 0.0 l_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, l_order, reason) + assert freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() l_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unexitable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() l_order['filled'] = 2 - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) - cancel_buy_order['status'] = 'open' - cancel_order_mock = MagicMock(return_value=cancel_buy_order) - trade.open_order_id = 'some_open_order' + cancel_entry_order['status'] = 'open' + cancel_order_mock = MagicMock(return_value=cancel_entry_order) + mocker.patch(f'{EXMS}.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) # min_pair_stake empty should not crash mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=None) - assert not freqtrade.handle_cancel_enter(trade, limit_order[entry_side(is_short)], reason) + assert not freqtrade.handle_cancel_enter( + trade, limit_order[entry_side(is_short)], trade.open_orders_ids[0], reason + ) + + # Retry ... + cbo = limit_order[entry_side(is_short)] + + mocker.patch('freqtrade.freqtradebot.sleep') + cbo['status'] = 'open' + co_mock = mocker.patch(f'{EXMS}.cancel_order_with_result', return_value=cbo) + fo_mock = mocker.patch(f'{EXMS}.fetch_order', return_value=cbo) + assert not freqtrade.handle_cancel_enter( + trade, cbo, cbo['id'], reason, replacing=True + ) + assert co_mock.call_count == 1 + assert fo_mock.call_count == 3 @pytest.mark.parametrize("is_short", [False, True]) @@ -3381,7 +3459,9 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf_usdt, is_sho trade = mock_trade_usdt_4(fee, is_short) Trade.session.add(trade) Trade.commit() - assert freqtrade.handle_cancel_enter(trade, limit_buy_order_canceled_empty, reason) + assert freqtrade.handle_cancel_enter( + trade, limit_buy_order_canceled_empty, trade.open_orders_ids[0], reason + ) assert cancel_order_mock.call_count == 0 assert log_has_re( f'{trade.entry_side.capitalize()} order fully cancelled. ' @@ -3418,7 +3498,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_order l_order['filled'] = 0.0 l_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_enter(trade, l_order, reason) + assert freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() @@ -3426,11 +3506,15 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf_usdt, limit_order order = deepcopy(l_order) order['status'] = 'canceled' mocker.patch(f'{EXMS}.fetch_order', return_value=order) - assert not freqtrade.handle_cancel_enter(trade, l_order, reason) + assert not freqtrade.handle_cancel_enter(trade, l_order, trade.open_orders_ids[0], reason) assert cancel_order_mock.call_count == 1 -def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: +@pytest.mark.parametrize('is_short', [True, False]) +@pytest.mark.parametrize('leverage', [1, 5]) +@pytest.mark.parametrize('amount', [2, 50]) +def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee, is_short, + leverage, amount) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -3438,7 +3522,9 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: EXMS, cancel_order=cancel_order_mock, ) - mocker.patch(f'{EXMS}.get_rate', return_value=0.245441) + entry_price = 0.245441 + + mocker.patch(f'{EXMS}.get_rate', return_value=entry_price) mocker.patch(f'{EXMS}.get_min_pair_stake_amount', return_value=0.2) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_order_fee') @@ -3446,29 +3532,30 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: freqtrade = FreqtradeBot(default_conf_usdt) trade = Trade( - pair='LTC/ETH', - amount=2, + pair='LTC/USDT', + amount=amount * leverage, exchange='binance', - open_rate=0.245441, - open_order_id="sell_123456", + open_rate=entry_price, open_date=dt_now() - timedelta(days=2), fee_open=fee.return_value, fee_close=fee.return_value, close_rate=0.555, close_date=dt_now(), exit_reason="sell_reason_whatever", - stake_amount=0.245441 * 2, + stake_amount=entry_price * amount, + leverage=leverage, + is_short=is_short, ) trade.orders = [ Order( - ft_order_side='buy', + ft_order_side=entry_side(is_short), ft_pair=trade.pair, ft_is_open=False, order_id='buy_123456', status="closed", symbol=trade.pair, order_type="market", - side="buy", + side=entry_side(is_short), price=trade.open_rate, average=trade.open_rate, filled=trade.amount, @@ -3478,14 +3565,14 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: order_filled_date=trade.open_date, ), Order( - ft_order_side='sell', + ft_order_side=exit_side(is_short), ft_pair=trade.pair, ft_is_open=True, order_id='sell_123456', status="open", symbol=trade.pair, order_type="limit", - side="sell", + side=exit_side(is_short), price=trade.open_rate, average=trade.open_rate, filled=0.0, @@ -3501,26 +3588,26 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] send_msg_mock.reset_mock() - assert freqtrade.handle_cancel_exit(trade, order, reason) + assert freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 assert trade.close_rate is None assert trade.exit_reason is None - assert trade.open_order_id is None + assert not trade.has_open_orders send_msg_mock.reset_mock() # Partial exit - below exit threshold - order['amount'] = 2 - order['filled'] = 1.9 - assert not freqtrade.handle_cancel_exit(trade, order, reason) + order['amount'] = amount * leverage + order['filled'] = amount * 0.99 * leverage + assert not freqtrade.handle_cancel_exit(trade, order, order['id'], reason) # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']) - assert not freqtrade.handle_cancel_exit(trade, order, reason) + assert not freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']) @@ -3531,8 +3618,8 @@ def test_handle_cancel_exit_limit(mocker, default_conf_usdt, fee) -> None: send_msg_mock.reset_mock() - order['filled'] = 1 - assert freqtrade.handle_cancel_exit(trade, order, reason) + order['filled'] = amount * 0.5 * leverage + assert freqtrade.handle_cancel_exit(trade, order, order['id'], reason) assert send_msg_mock.call_count == 1 assert (send_msg_mock.call_args_list[0][0][0]['reason'] == CANCEL_REASON['PARTIALLY_FILLED']) @@ -3548,17 +3635,16 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: # TODO: should not be magicmock trade = MagicMock() - trade.open_order_id = '125' + order_id = '125' reason = CANCEL_REASON['TIMEOUT'] order = {'remaining': 1, 'id': '125', 'amount': 1, 'status': "open"} - assert not freqtrade.handle_cancel_exit(trade, order, reason) + assert not freqtrade.handle_cancel_exit(trade, order, order_id, reason) # mocker.patch(f'{EXMS}.cancel_order_with_result', return_value=order) # assert not freqtrade.handle_cancel_exit(trade, order, reason) - # assert trade.open_order_id == '125' @pytest.mark.parametrize("is_short, open_rate, amt", [ @@ -4005,7 +4091,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit( freqtrade.exit_positions(trades) assert trade assert trade.stoploss_order_id == '123' - assert trade.open_order_id is None + assert not trade.has_open_orders # Assuming stoploss on exchange is hit # stoploss_order_id should become None @@ -4300,7 +4386,6 @@ def test__safe_exit_amount(default_conf_usdt, fee, caplog, mocker, amount_wallet amount=amount, exchange='binance', open_rate=0.245441, - open_order_id="123456", fee_open=fee.return_value, fee_close=fee.return_value, ) @@ -4643,7 +4728,6 @@ def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fe open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4671,7 +4755,6 @@ def test_get_real_amount_quote_dust(default_conf_usdt, trades_for_order, buy_ord open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4695,7 +4778,6 @@ def test_get_real_amount_no_trade(default_conf_usdt, buy_order_fee, caplog, mock open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4748,7 +4830,6 @@ def test_get_real_amount( fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4795,8 +4876,7 @@ def test_get_real_amount_multi( exchange='binance', fee_open=fee.return_value, fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" + open_rate=0.245441 ) # Fake markets entry to enable fee parsing @@ -4841,7 +4921,6 @@ def test_get_real_amount_invalid_order(default_conf_usdt, trades_for_order, buy_ fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4863,7 +4942,6 @@ def test_get_real_amount_fees_order(default_conf_usdt, market_buy_order_usdt_dou fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4890,7 +4968,6 @@ def test_get_real_amount_wrong_amount(default_conf_usdt, trades_for_order, buy_o open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4915,7 +4992,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_ord fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.245441, - open_order_id="123456" ) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -4934,7 +5010,6 @@ def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker): open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) order = { 'id': 'mocked_order', @@ -4994,8 +5069,7 @@ def test_get_real_amount_in_point(default_conf_usdt, buy_order_fee, fee, mocker, exchange='binance', fee_open=fee.return_value, fee_close=fee.return_value, - open_rate=0.245441, - open_order_id="123456" + open_rate=0.245441 ) limit_buy_order_usdt['amount'] = amount freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) @@ -5037,7 +5111,6 @@ def test_apply_fee_conditional(default_conf_usdt, fee, mocker, caplog, open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456" ) order = Order( ft_order_side='buy', @@ -5075,8 +5148,7 @@ def test_apply_fee_conditional_multibuy(default_conf_usdt, fee, mocker, caplog, exchange='binance', open_rate=0.245441, fee_open=fee.return_value, - fee_close=fee.return_value, - open_order_id="123456" + fee_close=fee.return_value ) # One closed order order = Order( @@ -5550,7 +5622,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap return_value={'status': 'open'}) def reset_open_orders(trade): - trade.open_order_id = None + trade.stoploss_order_id = None trade.is_short = is_short @@ -5562,7 +5634,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # No open order trade = trades[1] reset_open_orders(trade) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5572,7 +5644,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 0 assert mock_uts.call_count == 0 # No change to orderid - as update_trade_state is mocked - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None caplog.clear() @@ -5581,7 +5653,9 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open buy order trade = trades[3] reset_open_orders(trade) - assert trade.open_order_id is None + + # This part in not relevant anymore + # assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5590,7 +5664,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 1 # Found open buy order - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.stoploss_order_id is None caplog.clear() @@ -5599,7 +5673,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open stoploss order trade = trades[4] reset_open_orders(trade) - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5608,7 +5682,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 2 # stoploss_order_id is "refound" and added to the trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.stoploss_order_id is not None caplog.clear() @@ -5618,7 +5692,8 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap # Open sell order trade = trades[5] reset_open_orders(trade) - assert trade.open_order_id is None + # This part in not relevant anymore + # assert not trade.has_open_orders assert trade.stoploss_order_id is None freqtrade.handle_insufficient_funds(trade) @@ -5627,7 +5702,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert mock_fo.call_count == 1 assert mock_uts.call_count == 1 # sell-orderid is "refound" and added to the trade - assert trade.open_order_id == order['id'] + assert trade.open_orders_ids[0] == order['id'] assert trade.stoploss_order_id is None caplog.clear() @@ -5644,6 +5719,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.spy(freqtrade, 'update_trade_state') @@ -5654,21 +5730,18 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor exit_order, ]) - order_id = entry_order['id'] - trade = Trade( - open_order_id=order_id, - pair='ETH/USDT', - fee_open=0.001, - fee_close=0.001, - open_rate=entry_order['price'], - open_date=dt_now(), - stake_amount=entry_order['cost'], - amount=entry_order['amount'], - exchange="binance", - is_short=is_short, - leverage=1, - ) + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + ) trade.orders.append(Order.parse_from_ccxt_object( entry_order, 'ADA/USDT', entry_side(is_short)) @@ -5676,7 +5749,8 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor Trade.session.add(trade) freqtrade.handle_onexchange_order(trade) assert log_has_re(r"Found previously unknown order .*", caplog) - assert mock_uts.call_count == 1 + # Update trade state is called twice, once for the known and once for the unknown order. + assert mock_uts.call_count == 2 assert mock_fo.call_count == 1 trade = Trade.session.scalars(select(Trade)).first() @@ -5686,6 +5760,77 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor assert trade.exit_reason == ExitType.SOLD_ON_EXCHANGE.value +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_onexchange_order_exit(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mock_uts = mocker.spy(freqtrade, 'update_trade_state') + + entry_order = limit_order[entry_side(is_short)] + add_entry_order = deepcopy(entry_order) + add_entry_order.update({ + 'id': '_partial_entry_id', + 'amount': add_entry_order['amount'] / 1.5, + 'cost': add_entry_order['cost'] / 1.5, + 'filled': add_entry_order['filled'] / 1.5, + }) + + exit_order_part = deepcopy(limit_order[exit_side(is_short)]) + exit_order_part.update({ + 'id': 'some_random_partial_id', + 'amount': exit_order_part['amount'] / 2, + 'cost': exit_order_part['cost'] / 2, + 'filled': exit_order_part['filled'] / 2, + }) + exit_order = limit_order[exit_side(is_short)] + + # Orders intentionally in the wrong sequence + mock_fo = mocker.patch(f'{EXMS}.fetch_orders', return_value=[ + entry_order, + exit_order_part, + exit_order, + add_entry_order, + ]) + + trade = Trade( + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + is_open=True, + ) + + trade.orders = [ + Order.parse_from_ccxt_object(entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order_part, trade.pair, exit_side(is_short)), + Order.parse_from_ccxt_object(add_entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order, trade.pair, exit_side(is_short)), + ] + trade.recalc_trade_from_orders() + Trade.session.add(trade) + Trade.commit() + + freqtrade.handle_onexchange_order(trade) + # assert log_has_re(r"Found previously unknown order .*", caplog) + # Update trade state is called three times, once for every order + assert mock_uts.call_count == 4 + assert mock_fo.call_count == 1 + + trade = Trade.session.scalars(select(Trade)).first() + + assert len(trade.orders) == 4 + assert trade.is_open is True + assert trade.exit_reason is None + assert trade.amount == 5.0 + + def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -5981,7 +6126,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 @@ -5991,7 +6136,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 assert not trade.fee_updated('buy') @@ -6001,7 +6146,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == 11 assert trade.stake_amount == 110 assert not trade.fee_updated('buy') @@ -6029,7 +6174,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: assert len(orders) == 2 trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id == '651' + assert '651' in trade.open_orders_ids assert trade.open_rate == 11 assert trade.amount == 10 assert trade.stake_amount == 110 @@ -6066,7 +6211,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id == '651' + assert '651' in trade.open_orders_ids assert trade.open_rate == 11 assert trade.amount == 10 assert trade.stake_amount == 110 @@ -6103,7 +6248,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 9.90909090909 assert trade.amount == 22 assert pytest.approx(trade.stake_amount) == 218 @@ -6145,7 +6290,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 8.729729729729 assert trade.amount == 37 assert trade.stake_amount == 323 @@ -6183,7 +6328,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: # Assert trade is as expected (averaged dca) trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.is_open assert trade.amount == 22 assert trade.stake_amount == 192.05405405405406 @@ -6260,7 +6405,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount @@ -6270,7 +6415,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount assert not trade.fee_updated(trade.entry_side) @@ -6280,7 +6425,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.open_rate == bid assert trade.stake_amount == bid * amount assert not trade.fee_updated(trade.entry_side) @@ -6315,7 +6460,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == 50 assert trade.open_rate == 11 assert trade.stake_amount == 550 @@ -6357,7 +6502,7 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == 50 assert trade.open_rate == 11 assert trade.stake_amount == 550 @@ -6457,7 +6602,7 @@ def test_position_adjust3(mocker, default_conf_usdt, fee, data) -> None: assert trade if idx < len(data) - 1: assert trade.is_open is True - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.amount == result[0] assert trade.open_rate == result[1] assert trade.stake_amount == result[2] @@ -6470,7 +6615,7 @@ def test_position_adjust3(mocker, default_conf_usdt, fee, data) -> None: trade = Trade.session.scalars(select(Trade)).first() assert trade - assert trade.open_order_id is None + assert not trade.has_open_orders assert trade.is_open is False diff --git a/tests/test_integration.py b/tests/test_integration.py index 41265ae74..ee1d4bbb3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -103,7 +103,6 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, trade.orders.append(oobj) trade.stoploss_order_id = f"stop{idx}" - trade.open_order_id = None n = freqtrade.exit_positions(trades) assert n == 2 @@ -194,8 +193,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, mocker, balance_rati for trade in trades: assert pytest.approx(trade.stake_amount) == result1 - # Reset trade open order id's - trade.open_order_id = None + trades = Trade.get_open_trades() assert len(trades) == 5 bals = freqtrade.wallets.get_all_balances() @@ -386,7 +384,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) assert len(Trade.get_trades().all()) == 1 trade: Trade = Trade.get_trades().first() assert len(trade.orders) == 1 - assert trade.open_order_id is not None + assert trade.has_open_orders assert pytest.approx(trade.stake_amount) == 60 assert trade.open_rate == 1.96 assert trade.stop_loss_pct == -0.1 @@ -399,7 +397,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 1 - assert trade.open_order_id is not None + assert trade.has_open_orders assert pytest.approx(trade.stake_amount) == 60 # Cancel order and place new one @@ -407,7 +405,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 - assert trade.open_order_id is not None + assert trade.has_open_orders # Open rate is not adjusted yet assert trade.open_rate == 1.96 assert trade.stop_loss_pct == -0.1 @@ -421,7 +419,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 2 - assert trade.open_order_id is None + assert not trade.has_open_orders # Open rate is not adjusted yet assert trade.open_rate == 1.99 assert pytest.approx(trade.stake_amount) == 60 @@ -438,7 +436,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 3 - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.open_rate == 1.99 assert trade.orders[-1].price == 1.96 assert trade.orders[-1].cost == 120 * leverage @@ -449,7 +447,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 4 - assert trade.open_order_id is not None + assert trade.has_open_orders assert trade.open_rate == 1.99 assert pytest.approx(trade.stake_amount) == 60 assert trade.orders[-1].price == 1.95 @@ -463,7 +461,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) freqtrade.process() trade = Trade.get_trades().first() assert len(trade.orders) == 4 - assert trade.open_order_id is None + assert not trade.has_open_orders assert pytest.approx(trade.open_rate) == 1.963153456 assert trade.orders[-1].price == 1.95 assert pytest.approx(trade.orders[-1].cost) == 120 * leverage @@ -497,6 +495,88 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) assert freqtrade.strategy.adjust_entry_price.call_count == 0 +@pytest.mark.parametrize('leverage', [1, 2]) +@pytest.mark.parametrize("is_short", [False, True]) +def test_dca_order_adjust_entry_replace_fails( + default_conf_usdt, ticker_usdt, fee, mocker, caplog, is_short, leverage +) -> None: + spot = leverage == 1 + if not spot: + default_conf_usdt['trading_mode'] = 'futures' + default_conf_usdt['margin_mode'] = 'isolated' + default_conf_usdt['position_adjustment_enable'] = True + default_conf_usdt['max_open_trades'] = 2 + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mocker.patch.multiple( + EXMS, + fetch_ticker=ticker_usdt, + get_fee=fee, + get_funding_fees=MagicMock(return_value=0), + ) + + # no order fills. + mocker.patch(f'{EXMS}._dry_is_price_crossed', side_effect=[False, True]) + patch_get_signal(freqtrade, enter_short=is_short, enter_long=not is_short) + freqtrade.enter_positions() + + trades = Trade.session.scalars( + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() + assert len(trades) == 1 + + mocker.patch(f'{EXMS}._dry_is_price_crossed', return_value=False) + + # Timeout to not interfere + freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) + + # Create DCA order for 2nd trade (so we have 2 open orders on 2 trades) + # this 2nd order won't fill. + + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=20) + + freqtrade.process() + + assert freqtrade.strategy.adjust_trade_position.call_count == 1 + trades = Trade.session.scalars( + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() + assert len(trades) == 2 + + # We now have 2 orders open + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=2.05) + freqtrade.manage_open_orders() + trades = Trade.session.scalars( + select(Trade) + .where(Order.ft_is_open.is_(True)) + .where(Order.ft_order_side != "stoploss") + .where(Order.ft_trade_id == Trade.id) + ).all() + assert len(trades) == 2 + assert len(Order.get_open_orders()) == 2 + # Entry adjustment is called + assert freqtrade.strategy.adjust_entry_price.call_count == 2 + + # Attempt order replacement - fails. + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) + + entry_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_entry', + return_value=False) + msg = r"Could not replace order for.*" + assert not log_has_re(msg, caplog) + freqtrade.manage_open_orders() + + assert log_has_re(msg, caplog) + assert entry_mock.call_count == 2 + assert len(Trade.get_trades().all()) == 1 + assert len(Order.get_open_orders()) == 0 + + @pytest.mark.parametrize('leverage', [1, 2]) def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog, leverage) -> None: default_conf_usdt['position_adjustment_enable'] = True diff --git a/tests/test_misc.py b/tests/test_misc.py index e94e299fd..7de1adbbc 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -121,6 +121,8 @@ def test_safe_value_fallback(): assert safe_value_fallback(dict1, 'keyNo', 'keyNo') is None assert safe_value_fallback(dict1, 'keyNo', 'keyNo', 55) == 55 + assert safe_value_fallback(dict1, 'keyNo', default_value=55) == 55 + assert safe_value_fallback(dict1, 'keyNo', None, default_value=55) == 55 def test_safe_value_fallback2(): diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 6ce975732..b70065645 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,8 +3,8 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_utc, - format_ms_time, shorten_date) +from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_ts_def, dt_utc, + format_date, format_ms_time, shorten_date) def test_dt_now(): @@ -22,6 +22,13 @@ def test_dt_now(): assert dt_ts(now) == int(now.timestamp() * 1000) +def test_dt_ts_def(): + assert dt_ts_def(None) == 0 + assert dt_ts_def(None, 123) == 123 + assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc)) == 1683244800000 + assert dt_ts_def(datetime(2023, 5, 5, tzinfo=timezone.utc), 123) == 1683244800000 + + def test_dt_utc(): assert dt_utc(2023, 5, 5) == datetime(2023, 5, 5, tzinfo=timezone.utc) assert dt_utc(2023, 5, 5, 0, 0, 0, 555500) == datetime(2023, 5, 5, 0, 0, 0, 555500, @@ -70,3 +77,14 @@ def test_format_ms_time() -> None: # Date 2017-12-13 08:02:01 date_in_epoch_ms = 1513152121000 assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') + + +def test_format_date() -> None: + + date = datetime(2023, 9, 1, 5, 2, 3, 455555, tzinfo=timezone.utc) + assert format_date(date) == '2023-09-01 05:02:03' + assert format_date(None) == '' + + date = datetime(2021, 9, 30, 22, 59, 3, 455555, tzinfo=timezone.utc) + assert format_date(date) == '2021-09-30 22:59:03' + assert format_date(None) == ''