From d4272269007b3f1d7027ca7b4f5e1c99980666b1 Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:15:20 +0200 Subject: [PATCH 01/23] Update docker_quickstart.md --- docs/docker_quickstart.md | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 84c1d596a..6b48a7877 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -10,14 +10,14 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -To simplify running freqtrade, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). +To simplify running freqtrade, [`docker compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). -## Freqtrade with docker-compose +## Freqtrade with docker -Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage. +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage. !!! Note - - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user. + - The following section assumes that `docker` is installed and available to the logged in user. - All below commands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. ### Docker quick start @@ -31,13 +31,13 @@ cd ft_userdata/ curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml # Pull the freqtrade image -docker-compose pull +docker compose pull # Create user directory structure -docker-compose run --rm freqtrade create-userdir --userdir user_data +docker compose run --rm freqtrade create-userdir --userdir user_data # Create configuration - Requires answering interactive questions -docker-compose run --rm freqtrade new-config --config user_data/config.json +docker compose run --rm freqtrade new-config --config user_data/config.json ``` The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. @@ -64,7 +64,7 @@ The `SampleStrategy` is run by default. Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). ``` bash -docker-compose up -d +docker compose up -d ``` !!! Warning "Default configuration" @@ -84,27 +84,27 @@ You can now access the UI by typing localhost:8080 in your browser. #### Monitoring the bot -You can check for running instances with `docker-compose ps`. +You can check for running instances with `docker compose ps`. This should list the service `freqtrade` as `running`. If that's not the case, best check the logs (see next point). -#### Docker-compose logs +#### Docker compose logs Logs will be written to: `user_data/logs/freqtrade.log`. -You can also check the latest log with the command `docker-compose logs -f`. +You can also check the latest log with the command `docker compose logs -f`. #### Database The database will be located at: `user_data/tradesv3.sqlite` -#### Updating freqtrade with docker-compose +#### Updating freqtrade with docker -Updating freqtrade when using `docker-compose` is as simple as running the following 2 commands: +Updating freqtrade when using `docker` is as simple as running the following 2 commands: ``` bash # Download the latest image -docker-compose pull +docker compose pull # Restart the image -docker-compose up -d +docker compose up -d ``` This will first pull the latest image, and will then restart the container with the just pulled version. @@ -116,43 +116,43 @@ This will first pull the latest image, and will then restart the container with Advanced users may edit the docker-compose file further to include all possible options or arguments. -All freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. +All freqtrade arguments will be available by running `docker compose run --rm freqtrade `. -!!! Warning "`docker-compose` for trade commands" - Trade commands (`freqtrade trade <...>`) should not be ran via `docker-compose run` - but should use `docker-compose up -d` instead. +!!! Warning "`docker compose` for trade commands" + Trade commands (`freqtrade trade <...>`) should not be ran via `docker compose run` - but should use `docker compose up -d` instead. This makes sure that the container is properly started (including port forwardings) and will make sure that the container will restart after a system reboot. If you intend to use freqUI, please also ensure to adjust the [configuration accordingly](rest-api.md#configuration-with-docker), otherwise the UI will not be available. -!!! Note "`docker-compose run --rm`" +!!! Note "`docker compose run --rm`" Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). -??? Note "Using docker without docker-compose" - "`docker-compose run --rm`" will require a compose file to be provided. +??? Note "Using docker without docker" + "`docker compose run --rm`" will require a compose file to be provided. Some freqtrade commands that don't require authentication such as `list-pairs` can be run with "`docker run --rm`" instead. For example `docker run --rm freqtradeorg/freqtrade:stable list-pairs --exchange binance --quote BTC --print-json`. This can be useful for fetching exchange information to add to your `config.json` without affecting your running containers. -#### Example: Download data with docker-compose +#### Example: Download data with docker Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. ``` bash -docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h +docker compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h ``` Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. -#### Example: Backtest with docker-compose +#### Example: Backtest with docker Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: ``` bash -docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m +docker compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m ``` Head over to the [Backtesting Documentation](backtesting.md) to learn more. -### Additional dependencies with docker-compose +### Additional dependencies with docker If your strategy requires dependencies not included in the default image - it will be necessary to build the image on your host. For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.custom](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.custom) for an example). @@ -166,15 +166,15 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the dockerfile: "./Dockerfile." ``` -You can then run `docker-compose build --pull` to build the docker image, and run it using the commands described above. +You can then run `docker compose build --pull` to build the docker image, and run it using the commands described above. -### Plotting with docker-compose +### Plotting with docker Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. You can then use these commands as follows: ``` bash -docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805 +docker compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805 ``` The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. @@ -185,7 +185,7 @@ Freqtrade provides a docker-compose file which starts up a jupyter lab server. You can run this server using the following command: ``` bash -docker-compose -f docker/docker-compose-jupyter.yml up +docker compose -f docker/docker-compose-jupyter.yml up ``` This will create a docker-container running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. @@ -194,7 +194,7 @@ Please use the link that's printed in the console after startup for simplified l Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) up-to-date. ``` bash -docker-compose -f docker/docker-compose-jupyter.yml build --no-cache +docker compose -f docker/docker-compose-jupyter.yml build --no-cache ``` ## Troubleshooting From abcbe7a42153740208f7ac4186fdecb0bc45f3f4 Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:15:59 +0200 Subject: [PATCH 02/23] Update updating.md --- docs/updating.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/updating.md b/docs/updating.md index 893bc846e..1e5dc8ffe 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -6,14 +6,14 @@ To update your freqtrade installation, please use one of the below methods, corr Breaking changes / changed behavior will be documented in the changelog that is posted alongside every release. For the develop branch, please follow PR's to avoid being surprised by changes. -## docker-compose +## docker !!! Note "Legacy installations using the `master` image" We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable` ``` bash -docker-compose pull -docker-compose up -d +docker compose pull +docker compose up -d ``` ## Installation via setup script From 11d6d0be9e25b1fac1be5cde8addbedeaedf129f Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:22:07 +0200 Subject: [PATCH 03/23] Update sql_cheatsheet.md --- docs/sql_cheatsheet.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index c42cb5575..67c081d4c 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -13,12 +13,12 @@ Feel free to use a visual Database editor like SqliteBrowser if you feel more co sudo apt-get install sqlite3 ``` -### Using sqlite3 via docker-compose +### Using sqlite3 via docker The freqtrade docker image does contain sqlite3, so you can edit the database without having to install anything on the host system. ``` bash -docker-compose exec freqtrade /bin/bash +docker compose exec freqtrade /bin/bash sqlite3 .sqlite ``` From fe3d99b5685ad681347a448681f045c05e7f541e Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:22:49 +0200 Subject: [PATCH 04/23] Update feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a18915462..db335bf09 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -18,7 +18,7 @@ Have you search for this feature before requesting it? It's highly likely that a * Operating system: ____ * Python Version: _____ (`python -V`) * CCXT version: _____ (`pip freeze | grep ccxt`) - * Freqtrade Version: ____ (`freqtrade -V` or `docker-compose run --rm freqtrade -V` for Freqtrade running in docker) + * Freqtrade Version: ____ (`freqtrade -V` or `docker compose run --rm freqtrade -V` for Freqtrade running in docker) ## Describe the enhancement From 67850d92af1c81dfc139b7045ac33f4e2056e9fb Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:24:46 +0200 Subject: [PATCH 05/23] Update question.md --- .github/ISSUE_TEMPLATE/question.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 4b02e5f19..9283f0e4f 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -18,7 +18,7 @@ Please do not use the question template to report bugs or to request new feature * Operating system: ____ * Python Version: _____ (`python -V`) * CCXT version: _____ (`pip freeze | grep ccxt`) - * Freqtrade Version: ____ (`freqtrade -V` or `docker-compose run --rm freqtrade -V` for Freqtrade running in docker) + * Freqtrade Version: ____ (`freqtrade -V` or `docker compose run --rm freqtrade -V` for Freqtrade running in docker) ## Your question From 35cc6aa966cc509cc9dc528c813928be4e6f3757 Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:25:37 +0200 Subject: [PATCH 06/23] Update data-analysis.md --- docs/data-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 926ed3eae..5f01ae38f 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -5,7 +5,7 @@ You can analyze the results of backtests and trading history easily using Jupyte ## Quick start with docker Freqtrade provides a docker-compose file which starts up a jupyter lab server. -You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up` +You can run this server using the following command: `docker compose -f docker/docker-compose-jupyter.yml up` This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. Please use the link that's printed in the console after startup for simplified login. From 8c39b37223ec3ea449bf277957794f19a5bd9c58 Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:26:09 +0200 Subject: [PATCH 07/23] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 54c9eab50..8637c0d68 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -20,7 +20,7 @@ Please do not use bug reports to request new features. * Operating system: ____ * Python Version: _____ (`python -V`) * CCXT version: _____ (`pip freeze | grep ccxt`) - * Freqtrade Version: ____ (`freqtrade -V` or `docker-compose run --rm freqtrade -V` for Freqtrade running in docker) + * Freqtrade Version: ____ (`freqtrade -V` or `docker compose run --rm freqtrade -V` for Freqtrade running in docker) Note: All issues other than enhancement requests will be closed without further comment if the above template is deleted or not filled out. From 51b410ac1a333e5ae744e68be13b5dca8b3a1748 Mon Sep 17 00:00:00 2001 From: Matteo Manzi <33622899+matteoettam09@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:28:29 +0200 Subject: [PATCH 08/23] Update utils.md --- docs/utils.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index ee8793159..2f4604323 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -654,7 +654,7 @@ Common arguments: You can also use webserver mode via docker. Starting a one-off container requires the configuration of the port explicitly, as ports are not exposed by default. -You can use `docker-compose run --rm -p 127.0.0.1:8080:8080 freqtrade webserver` to start a one-off container that'll be removed once you stop it. This assumes that port 8080 is still available and no other bot is running on that port. +You can use `docker compose run --rm -p 127.0.0.1:8080:8080 freqtrade webserver` to start a one-off container that'll be removed once you stop it. This assumes that port 8080 is still available and no other bot is running on that port. Alternatively, you can reconfigure the docker-compose file to have the command updated: @@ -664,7 +664,7 @@ Alternatively, you can reconfigure the docker-compose file to have the command u --config /freqtrade/user_data/config.json ``` -You can now use `docker-compose up` to start the webserver. +You can now use `docker compose up` to start the webserver. This assumes that the configuration has a webserver enabled and configured for docker (listening port = `0.0.0.0`). !!! Tip From eb81cccedebb79bd363b4d8fb48b49b6700e9749 Mon Sep 17 00:00:00 2001 From: k <> Date: Thu, 1 Dec 2022 16:37:24 +0800 Subject: [PATCH 09/23] add download-data command change directory fix relative config path --- .../templates/strategy_analysis_example.ipynb | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 5fb14ab2f..f7d68b41c 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -7,7 +7,7 @@ "# Strategy analysis example\n", "\n", "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data.\n", - "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location." + "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location, using command like `freqtrade download-data --exchange binance --trading-mod spot --pairs BTC/USDT --days 7 -t 5m`." ] }, { @@ -23,7 +23,21 @@ "metadata": {}, "outputs": [], "source": [ + "import os\n", "from pathlib import Path\n", + "\n", + "# Change current working directory from `somedir/freqtrade/user_data/notebooks` to project root `somedir/freqtrade`, so relative paths remain consistent.\n", + "if not Path(\"LICENSE\").is_file():\n", + " os.chdir(\"../../\")\n", + "print(Path.cwd())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "from freqtrade.configuration import Configuration\n", "\n", "# Customize these according to your needs.\n", @@ -31,14 +45,14 @@ "# Initialize empty configuration object\n", "config = Configuration.from_files([])\n", "# Optionally (recommended), use existing configuration file\n", - "# config = Configuration.from_files([\"config.json\"])\n", + "# config = Configuration.from_files([\"user_data/config.json\"])\n", "\n", "# Define some constants\n", "config[\"timeframe\"] = \"5m\"\n", "# Name of the strategy class\n", "config[\"strategy\"] = \"SampleStrategy\"\n", "# Location of the data\n", - "data_location = config['datadir']\n", + "data_location = config[\"datadir\"]\n", "# Pair to analyze - Only use one pair here\n", "pair = \"BTC/USDT\"" ] @@ -56,7 +70,7 @@ "candles = load_pair_history(datadir=data_location,\n", " timeframe=config[\"timeframe\"],\n", " pair=pair,\n", - " data_format = \"hdf5\",\n", + " data_format = \"json\",\n", " candle_type=CandleType.SPOT,\n", " )\n", "\n", @@ -365,7 +379,7 @@ "metadata": { "file_extension": ".py", "kernelspec": { - "display_name": "Python 3.9.7 64-bit ('trade_397')", + "display_name": "Python 3.11.0 64-bit", "language": "python", "name": "python3" }, @@ -379,7 +393,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.0" }, "mimetype": "text/x-python", "name": "python", @@ -430,7 +444,7 @@ "version": 3, "vscode": { "interpreter": { - "hash": "675f32a300d6d26767470181ad0b11dd4676bcce7ed1dd2ffe2fbc370c95fc7c" + "hash": "945ba00099661281427cc644a7000ee9eeea5ce6ad3bf937939d3d384b8f3881" } } }, From e734b399296cd88e77d6962281f13f49a9a9b016 Mon Sep 17 00:00:00 2001 From: Emre Date: Mon, 5 Dec 2022 14:54:42 +0300 Subject: [PATCH 10/23] Make model_training_parameters optional --- config_examples/config_freqai.example.json | 4 +--- docs/freqai-configuration.md | 11 ++++------- freqtrade/constants.py | 5 ++--- .../freqai/prediction_models/ReinforcementLearner.py | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 5e564a1fc..dfd54b3d9 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -79,9 +79,7 @@ "test_size": 0.33, "random_state": 1 }, - "model_training_parameters": { - "n_estimators": 1000 - } + "model_training_parameters": {} }, "bot_name": "", "force_entry_enable": true, diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 5c3bbf90c..10f5838c9 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -26,10 +26,7 @@ FreqAI is configured through the typical [Freqtrade config file](configuration.m }, "data_split_parameters" : { "test_size": 0.25 - }, - "model_training_parameters" : { - "n_estimators": 100 - }, + } } ``` @@ -118,7 +115,7 @@ The FreqAI strategy requires including the following lines of code in the standa ``` -Notice how the `populate_any_indicators()` is where [features](freqai-feature-engineering.md#feature-engineering) and labels/targets are added. A full example strategy is available in `templates/FreqaiExampleStrategy.py`. +Notice how the `populate_any_indicators()` is where [features](freqai-feature-engineering.md#feature-engineering) and labels/targets are added. A full example strategy is available in `templates/FreqaiExampleStrategy.py`. Notice also the location of the labels under `if set_generalized_indicators:` at the bottom of the example. This is where single features and labels/targets should be added to the feature set to avoid duplication of them from various configuration parameters that multiply the feature set, such as `include_timeframes`. @@ -182,7 +179,7 @@ The `startup_candle_count` in the FreqAI strategy needs to be set up in the same ## Creating a dynamic target threshold -Deciding when to enter or exit a trade can be done in a dynamic way to reflect current market conditions. FreqAI allows you to return additional information from the training of a model (more info [here](freqai-feature-engineering.md#returning-additional-info-from-training)). For example, the `&*_std/mean` return values describe the statistical distribution of the target/label *during the most recent training*. Comparing a given prediction to these values allows you to know the rarity of the prediction. In `templates/FreqaiExampleStrategy.py`, the `target_roi` and `sell_roi` are defined to be 1.25 z-scores away from the mean which causes predictions that are closer to the mean to be filtered out. +Deciding when to enter or exit a trade can be done in a dynamic way to reflect current market conditions. FreqAI allows you to return additional information from the training of a model (more info [here](freqai-feature-engineering.md#returning-additional-info-from-training)). For example, the `&*_std/mean` return values describe the statistical distribution of the target/label *during the most recent training*. Comparing a given prediction to these values allows you to know the rarity of the prediction. In `templates/FreqaiExampleStrategy.py`, the `target_roi` and `sell_roi` are defined to be 1.25 z-scores away from the mean which causes predictions that are closer to the mean to be filtered out. ```python dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25 @@ -230,7 +227,7 @@ If you want to predict multiple targets, you need to define multiple labels usin #### Classifiers -If you are using a classifier, you need to specify a target that has discrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set +If you are using a classifier, you need to specify a target that has discrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set ```python df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d869b89f6..ca1be1d6a 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -608,9 +608,8 @@ CONF_SCHEMA = { "backtest_period_days", "identifier", "feature_parameters", - "data_split_parameters", - "model_training_parameters" - ] + "data_split_parameters" + ] }, }, } diff --git a/freqtrade/freqai/prediction_models/ReinforcementLearner.py b/freqtrade/freqai/prediction_models/ReinforcementLearner.py index 61b01e21b..39901859c 100644 --- a/freqtrade/freqai/prediction_models/ReinforcementLearner.py +++ b/freqtrade/freqai/prediction_models/ReinforcementLearner.py @@ -61,7 +61,7 @@ class ReinforcementLearner(BaseReinforcementLearningModel): model = self.MODELCLASS(self.policy_type, self.train_env, policy_kwargs=policy_kwargs, tensorboard_log=Path( dk.full_path / "tensorboard" / dk.pair.split('/')[0]), - **self.freqai_info['model_training_parameters'] + **self.freqai_info.get('model_training_parameters', {}) ) else: logger.info('Continual training activated - starting training from previously ' From 730fba956b55b67555bf5766532faf7ddc8ba856 Mon Sep 17 00:00:00 2001 From: Emre Date: Mon, 5 Dec 2022 16:16:17 +0300 Subject: [PATCH 11/23] Ensure base tf included in include_timeframes --- freqtrade/freqai/utils.py | 20 ++++++++++++++++++++ freqtrade/strategy/interface.py | 4 +++- tests/freqai/test_freqai_interface.py | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 806e3ca15..7a9d3df76 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -233,3 +233,23 @@ def get_timerange_backtest_live_models(config: Config) -> str: dd = FreqaiDataDrawer(models_path, config) timerange = dd.get_timerange_from_live_historic_predictions() return timerange.timerange_str + + +def ensure_base_tf_in_include_timeframes(config: Config) -> Config: + """ + Ensure that the base timeframe is included in the include_timeframes list + :param config: Configuration dictionary + + :return config: Configuration dictionary + """ + feature_parameters = config.get('freqai', {}).get('feature_parameters', {}) + include_timeframes = feature_parameters.get('include_timeframes', []) + + if config['timeframe'] in include_timeframes: + return config + + include_timeframes = [config['timeframe']] + include_timeframes + config.get('freqai', {}).get('feature_parameters', {}) \ + .update({**feature_parameters, 'include_timeframes': include_timeframes}) + + return config diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 681c5fcbb..48a03e216 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,9 +148,11 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.utils import download_all_data_for_training + from freqtrade.freqai.utils import (download_all_data_for_training, + ensure_base_tf_in_include_timeframes) from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) + self.config = ensure_base_tf_in_include_timeframes(self.config) self.freqai_info = self.config["freqai"] # download the desired data in dry/live diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index c53137093..6f01c66f6 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -9,7 +9,9 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.utils import download_all_data_for_training, get_required_data_timerange +from freqtrade.freqai.utils import (download_all_data_for_training, + ensure_base_tf_in_include_timeframes, + get_required_data_timerange) from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager @@ -528,6 +530,19 @@ def test_start_set_train_queue(mocker, freqai_conf, caplog): ) +def test_base_tf_in_include_timeframes(mocker, freqai_conf): + freqai_conf['timeframe'] = '5m' + freqai_conf['freqai']['feature_parameters'].update({ + 'include_timeframes': ['15m', '1h'] + }) + updated_conf = ensure_base_tf_in_include_timeframes(freqai_conf) + assert updated_conf['freqai']['feature_parameters']['include_timeframes'] == [ + '5m', '15m', '1h', + ] + last_conf = ensure_base_tf_in_include_timeframes(updated_conf) + assert last_conf == updated_conf + + def test_get_required_data_timerange(mocker, freqai_conf): time_range = get_required_data_timerange(freqai_conf) assert (time_range.stopts - time_range.startts) == 177300 From 189fa64052b0261a272765ed799941e8e001ef4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 5 Dec 2022 18:14:16 +0100 Subject: [PATCH 12/23] Add more dynamic to directory change --- docs/strategy_analysis_example.md | 33 +++++++++++++-- .../templates/strategy_analysis_example.ipynb | 40 ++++++++++++++----- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index bae4a9108..e3d2870e2 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -2,12 +2,37 @@ Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data. The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location. +Please follow the [documentation](https://www.freqtrade.io/en/stable/data-download/) for more details. ## Setup +### Change Working directory to repository root + ```python +import os from pathlib import Path + +# Change directory +# Modify this cell to insure that the output shows the correct path. +# Define all paths relative to the project root shown in the cell output +project_root = "somedir/freqtrade" +i=0 +try: + os.chdirdir(project_root) + assert Path('LICENSE').is_file() +except: + while i<4 and (not Path('LICENSE').is_file()): + os.chdir(Path(Path.cwd(), '../')) + i+=1 + project_root = Path.cwd() +print(Path.cwd()) +``` + +### Configure Freqtrade environment + + +```python from freqtrade.configuration import Configuration # Customize these according to your needs. @@ -15,14 +40,14 @@ from freqtrade.configuration import Configuration # Initialize empty configuration object config = Configuration.from_files([]) # Optionally (recommended), use existing configuration file -# config = Configuration.from_files(["config.json"]) +# config = Configuration.from_files(["user_data/config.json"]) # Define some constants config["timeframe"] = "5m" # Name of the strategy class config["strategy"] = "SampleStrategy" # Location of the data -data_location = config['datadir'] +data_location = config["datadir"] # Pair to analyze - Only use one pair here pair = "BTC/USDT" ``` @@ -36,12 +61,12 @@ from freqtrade.enums import CandleType candles = load_pair_history(datadir=data_location, timeframe=config["timeframe"], pair=pair, - data_format = "hdf5", + data_format = "json", # Make sure to update this to your data candle_type=CandleType.SPOT, ) # Confirm success -print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}") +print(f"Loaded {len(candles)} rows of data for {pair} from {data_location}") candles.head() ``` diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index f7d68b41c..dfbcedb72 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -7,14 +7,17 @@ "# Strategy analysis example\n", "\n", "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data.\n", - "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location, using command like `freqtrade download-data --exchange binance --trading-mod spot --pairs BTC/USDT --days 7 -t 5m`." + "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location.\n", + "Please follow the [documentation](https://www.freqtrade.io/en/stable/data-download/) for more details." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Setup" + "## Setup\n", + "\n", + "### Change Working directory to repository root" ] }, { @@ -26,12 +29,29 @@ "import os\n", "from pathlib import Path\n", "\n", - "# Change current working directory from `somedir/freqtrade/user_data/notebooks` to project root `somedir/freqtrade`, so relative paths remain consistent.\n", - "if not Path(\"LICENSE\").is_file():\n", - " os.chdir(\"../../\")\n", + "# Change directory\n", + "# Modify this cell to insure that the output shows the correct path.\n", + "# Define all paths relative to the project root shown in the cell output\n", + "project_root = \"somedir/freqtrade\"\n", + "i=0\n", + "try:\n", + " os.chdirdir(project_root)\n", + " assert Path('LICENSE').is_file()\n", + "except:\n", + " while i<4 and (not Path('LICENSE').is_file()):\n", + " os.chdir(Path(Path.cwd(), '../'))\n", + " i+=1\n", + " project_root = Path.cwd()\n", "print(Path.cwd())" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configure Freqtrade environment" + ] + }, { "cell_type": "code", "execution_count": null, @@ -70,12 +90,12 @@ "candles = load_pair_history(datadir=data_location,\n", " timeframe=config[\"timeframe\"],\n", " pair=pair,\n", - " data_format = \"json\",\n", + " data_format = \"json\", # Make sure to update this to your data\n", " candle_type=CandleType.SPOT,\n", " )\n", "\n", "# Confirm success\n", - "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n", + "print(f\"Loaded {len(candles)} rows of data for {pair} from {data_location}\")\n", "candles.head()" ] }, @@ -379,7 +399,7 @@ "metadata": { "file_extension": ".py", "kernelspec": { - "display_name": "Python 3.11.0 64-bit", + "display_name": "Python 3.9.7 64-bit", "language": "python", "name": "python3" }, @@ -393,7 +413,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.9.7" }, "mimetype": "text/x-python", "name": "python", @@ -444,7 +464,7 @@ "version": 3, "vscode": { "interpreter": { - "hash": "945ba00099661281427cc644a7000ee9eeea5ce6ad3bf937939d3d384b8f3881" + "hash": "675f32a300d6d26767470181ad0b11dd4676bcce7ed1dd2ffe2fbc370c95fc7c" } } }, From bc48099e48333d5c657fcbb11831ea8cd1700697 Mon Sep 17 00:00:00 2001 From: Emre Date: Mon, 5 Dec 2022 23:52:48 +0300 Subject: [PATCH 13/23] Revert changes --- freqtrade/freqai/utils.py | 20 -------------------- freqtrade/strategy/interface.py | 4 +--- tests/freqai/test_freqai_interface.py | 17 +---------------- 3 files changed, 2 insertions(+), 39 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 7a9d3df76..806e3ca15 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -233,23 +233,3 @@ def get_timerange_backtest_live_models(config: Config) -> str: dd = FreqaiDataDrawer(models_path, config) timerange = dd.get_timerange_from_live_historic_predictions() return timerange.timerange_str - - -def ensure_base_tf_in_include_timeframes(config: Config) -> Config: - """ - Ensure that the base timeframe is included in the include_timeframes list - :param config: Configuration dictionary - - :return config: Configuration dictionary - """ - feature_parameters = config.get('freqai', {}).get('feature_parameters', {}) - include_timeframes = feature_parameters.get('include_timeframes', []) - - if config['timeframe'] in include_timeframes: - return config - - include_timeframes = [config['timeframe']] + include_timeframes - config.get('freqai', {}).get('feature_parameters', {}) \ - .update({**feature_parameters, 'include_timeframes': include_timeframes}) - - return config diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 48a03e216..681c5fcbb 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -148,11 +148,9 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: if self.config.get('freqai', {}).get('enabled', False): # Import here to avoid importing this if freqAI is disabled - from freqtrade.freqai.utils import (download_all_data_for_training, - ensure_base_tf_in_include_timeframes) + from freqtrade.freqai.utils import download_all_data_for_training from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) - self.config = ensure_base_tf_in_include_timeframes(self.config) self.freqai_info = self.config["freqai"] # download the desired data in dry/live diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 6f01c66f6..c53137093 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -9,9 +9,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.utils import (download_all_data_for_training, - ensure_base_tf_in_include_timeframes, - get_required_data_timerange) +from freqtrade.freqai.utils import download_all_data_for_training, get_required_data_timerange from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager @@ -530,19 +528,6 @@ def test_start_set_train_queue(mocker, freqai_conf, caplog): ) -def test_base_tf_in_include_timeframes(mocker, freqai_conf): - freqai_conf['timeframe'] = '5m' - freqai_conf['freqai']['feature_parameters'].update({ - 'include_timeframes': ['15m', '1h'] - }) - updated_conf = ensure_base_tf_in_include_timeframes(freqai_conf) - assert updated_conf['freqai']['feature_parameters']['include_timeframes'] == [ - '5m', '15m', '1h', - ] - last_conf = ensure_base_tf_in_include_timeframes(updated_conf) - assert last_conf == updated_conf - - def test_get_required_data_timerange(mocker, freqai_conf): time_range = get_required_data_timerange(freqai_conf) assert (time_range.stopts - time_range.startts) == 177300 From 26a61afa15bce5d85256e8706534295f8cb033c3 Mon Sep 17 00:00:00 2001 From: Emre Date: Mon, 5 Dec 2022 23:54:15 +0300 Subject: [PATCH 14/23] Move base tf logic to config validation --- freqtrade/configuration/config_validation.py | 7 +++++++ tests/test_configuration.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index bf0657994..606f081ef 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -355,6 +355,13 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None: f"Main timeframe of {main_tf} must be smaller or equal to FreqAI " f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}") + # Ensure that the base timeframe is included in the include_timeframes list + if main_tf not in freqai_include_timeframes: + feature_parameters = conf.get('freqai', {}).get('feature_parameters', {}) + include_timeframes = [main_tf] + freqai_include_timeframes + conf.get('freqai', {}).get('feature_parameters', {}) \ + .update({**feature_parameters, 'include_timeframes': include_timeframes}) + def _validate_freqai_backtest(conf: Dict[str, Any]) -> None: if conf.get('runmode', RunMode.OTHER) == RunMode.BACKTEST: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 1bcff20db..cdf9f2f2e 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1046,8 +1046,13 @@ def test__validate_freqai_include_timeframes(default_conf, caplog) -> None: # Validation pass conf.update({'timeframe': '1m'}) validate_config_consistency(conf) - conf.update({'analyze_per_epoch': True}) + # Ensure base timeframe is in include_timeframes + conf['freqai']['feature_parameters']['include_timeframes'] = ["5m", "15m"] + validate_config_consistency(conf) + assert conf['freqai']['feature_parameters']['include_timeframes'] == ["1m", "5m", "15m"] + + conf.update({'analyze_per_epoch': True}) with pytest.raises(OperationalException, match=r"Using analyze-per-epoch .* not supported with a FreqAI strategy."): validate_config_consistency(conf) From 227cdb09386153fd7a871e3b72ff46cd2999962e Mon Sep 17 00:00:00 2001 From: Emre Date: Mon, 5 Dec 2022 23:58:04 +0300 Subject: [PATCH 15/23] Change dict update order --- freqtrade/configuration/config_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 606f081ef..7e291cb90 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -360,7 +360,7 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None: feature_parameters = conf.get('freqai', {}).get('feature_parameters', {}) include_timeframes = [main_tf] + freqai_include_timeframes conf.get('freqai', {}).get('feature_parameters', {}) \ - .update({**feature_parameters, 'include_timeframes': include_timeframes}) + .update({'include_timeframes': include_timeframes, **feature_parameters}) def _validate_freqai_backtest(conf: Dict[str, Any]) -> None: From 9b4364ddc3e410ca445cf08d73c606aed4323e6d Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 7 Dec 2022 19:49:14 +0100 Subject: [PATCH 16/23] ensure that add_state_info is deactivated during backtesting --- freqtrade/freqai/RL/BaseEnvironment.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index a31ded0c6..c217b72dd 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -12,6 +12,7 @@ from gym.utils import seeding from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import RunMode logger = logging.getLogger(__name__) @@ -78,6 +79,11 @@ class BaseEnvironment(gym.Env): # set here to default 5Ac, but all children envs can override this self.actions: Type[Enum] = BaseActions self.custom_info: dict = {} + self.live: bool = False + if dp: + self.live = dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE) + if not self.live and self.add_state_info: + logger.warning("add_state_info is not available in backtesting. Deactivating.") def reset_env(self, df: DataFrame, prices: DataFrame, window_size: int, reward_kwargs: dict, starting_point=True): @@ -188,7 +194,7 @@ class BaseEnvironment(gym.Env): """ features_window = self.signal_features[( self._current_tick - self.window_size):self._current_tick] - if self.add_state_info: + if self.add_state_info and self.live: features_and_state = DataFrame(np.zeros((len(features_window), 3)), columns=['current_profit_pct', 'position', From 7b3406914c2a219b877867e08f93c26ab64d9e41 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 7 Dec 2022 19:49:39 +0100 Subject: [PATCH 17/23] flip add_state_info --- freqtrade/freqai/RL/BaseEnvironment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/RL/BaseEnvironment.py b/freqtrade/freqai/RL/BaseEnvironment.py index c217b72dd..86c63c382 100644 --- a/freqtrade/freqai/RL/BaseEnvironment.py +++ b/freqtrade/freqai/RL/BaseEnvironment.py @@ -83,6 +83,7 @@ class BaseEnvironment(gym.Env): if dp: self.live = dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE) if not self.live and self.add_state_info: + self.add_state_info = False logger.warning("add_state_info is not available in backtesting. Deactivating.") def reset_env(self, df: DataFrame, prices: DataFrame, window_size: int, From 74e623fe5b4c5931362f149ce88d52ed3cb12cdc Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Dec 2022 08:33:07 +0100 Subject: [PATCH 18/23] Improve kraken test resiliance --- tests/exchange/test_ccxt_compat.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 280876ae8..7f23c2031 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -224,8 +224,13 @@ class TestCCXTExchange(): for val in [1, 2, 5, 25, 100]: l2 = exchange.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: - assert len(l2['asks']) == val - assert len(l2['bids']) == val + if val > 50: + # Orderbooks are not always this deep. + assert val - 5 < len(l2['asks']) <= val + assert val - 5 < len(l2['bids']) <= val + else: + assert len(l2['asks']) == val + assert len(l2['bids']) == val else: next_limit = exchange.get_next_limit_in_list( val, l2_limit_range, l2_limit_range_required) From 3d3a7033ed34f8c9bed86c729198e8a4b5e0414f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Dec 2022 08:46:16 +0100 Subject: [PATCH 19/23] Improve Docker documentation wording --- docs/docker_quickstart.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 6b48a7877..89f737d71 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -4,13 +4,15 @@ This page explains how to run the bot with Docker. It is not meant to work out o ## Install Docker -Start by downloading and installing Docker CE for your platform: +Start by downloading and installing Docker / Docker Desktop for your platform: * [Mac](https://docs.docker.com/docker-for-mac/install/) * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -To simplify running freqtrade, [`docker compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). +!!! Info "Docker compose install" + Freqtrade documentation assumes the use of Docker desktop (or the docker compose plugin). + While the docker-compose standalone installation still works, it will require changing all `docker compose` commands from `docker compose` to `docker-compose` to work (e.g. `docker compose up -d` will become `docker-compose up -d`). ## Freqtrade with docker From bbedc4b63efd08a4e4e3b2371a8463e6f6e445b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Dec 2022 14:15:29 +0100 Subject: [PATCH 20/23] Stop clock to avoid random failures on slow CI runs --- tests/rpc/test_rpc_telegram.py | 70 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3552d5fe7..1f4665867 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -12,6 +12,7 @@ from unittest.mock import ANY, MagicMock import arrow import pytest +import time_machine from pandas import DataFrame from telegram import Chat, Message, ReplyKeyboardMarkup, Update from telegram.error import BadRequest, NetworkError, TelegramError @@ -2065,41 +2066,42 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, default_conf['telegram']['notification_settings']['exit_fill'] = 'on' telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) - telegram.send_msg({ - 'type': RPCMessageType.EXIT_FILL, - 'trade_id': 1, - 'exchange': 'Binance', - 'pair': 'KEY/ETH', - 'leverage': leverage, - 'direction': direction, - 'gain': 'loss', - 'limit': 3.201e-05, - 'amount': 1333.3333333333335, - 'order_type': 'market', - 'open_rate': 7.5e-05, - 'close_rate': 3.201e-05, - 'profit_amount': -0.05746268, - 'profit_ratio': -0.57405275, - 'stake_currency': 'ETH', - 'enter_tag': enter_signal, - 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), - }) + with time_machine.travel("2022-09-01 05:00:00 +00:00", tick=False) as t: + telegram.send_msg({ + 'type': RPCMessageType.EXIT_FILL, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'leverage': leverage, + 'direction': direction, + 'gain': 'loss', + 'limit': 3.201e-05, + 'amount': 1333.3333333333335, + 'order_type': 'market', + 'open_rate': 7.5e-05, + 'close_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_ratio': -0.57405275, + 'stake_currency': 'ETH', + 'enter_tag': enter_signal, + 'exit_reason': ExitType.STOP_LOSS.value, + 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'close_date': arrow.utcnow(), + }) - leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' - assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' - '*Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' - f'*Enter Tag:* `{enter_signal}`\n' - '*Exit Reason:* `stop_loss`\n' - f"*Direction:* `{direction}`\n" - f"{leverage_text}" - '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Exit Rate:* `0.00003201`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`' - ) + leverage_text = f'*Leverage:* `{leverage}`\n' if leverage and leverage != 1.0 else '' + assert msg_mock.call_args[0][0] == ( + '\N{WARNING SIGN} *Binance (dry):* Exited KEY/ETH (#1)\n' + '*Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' + f'*Enter Tag:* `{enter_signal}`\n' + '*Exit Reason:* `stop_loss`\n' + f"*Direction:* `{direction}`\n" + f"{leverage_text}" + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`' + ) def test_send_msg_status_notification(default_conf, mocker) -> None: From 1da8ad69d9501838fee5792b39563d9925ed7ad5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Dec 2022 14:33:07 +0100 Subject: [PATCH 21/23] improve more tests by freezing time --- tests/rpc/test_rpc_telegram.py | 223 +++++++++++++++++---------------- 1 file changed, 112 insertions(+), 111 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 1f4665867..58977a94a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1907,119 +1907,120 @@ def test_send_msg_entry_fill_notification(default_conf, mocker, message_type, en def test_send_msg_sell_notification(default_conf, mocker) -> None: - telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + with time_machine.travel("2022-09-01 05:00:00 +00:00", tick=False): + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) - old_convamount = telegram._rpc._fiat_converter.convert_amount - telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812 - telegram.send_msg({ - 'type': RPCMessageType.EXIT, - 'trade_id': 1, - 'exchange': 'Binance', - 'pair': 'KEY/ETH', - 'leverage': 1.0, - 'direction': 'Long', - 'gain': 'loss', - 'order_rate': 3.201e-05, - 'amount': 1333.3333333333335, - 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, - 'profit_amount': -0.05746268, - 'profit_ratio': -0.57405275, - 'stake_currency': 'ETH', - 'fiat_currency': 'USD', - 'enter_tag': 'buy_signal1', - 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(hours=-1), - 'close_date': arrow.utcnow(), - }) - assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' - '*Enter Tag:* `buy_signal1`\n' - '*Exit Reason:* `stop_loss`\n' - '*Direction:* `Long`\n' - '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' - '*Duration:* `1:00:00 (60.0 min)`' - ) - - msg_mock.reset_mock() - telegram.send_msg({ - 'type': RPCMessageType.EXIT, - 'trade_id': 1, - 'exchange': 'Binance', - 'pair': 'KEY/ETH', - 'direction': 'Long', - 'gain': 'loss', - 'order_rate': 3.201e-05, - 'amount': 1333.3333333333335, - 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, - 'cumulative_profit': -0.15746268, - 'profit_amount': -0.05746268, - 'profit_ratio': -0.57405275, - 'stake_currency': 'ETH', - 'fiat_currency': 'USD', - 'enter_tag': 'buy_signal1', - 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), - 'stake_amount': 0.01, - 'sub_trade': True, - }) - assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Sub Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' - '*Cumulative Profit:* (`-0.15746268 ETH / -24.812 USD`)\n' - '*Enter Tag:* `buy_signal1`\n' - '*Exit Reason:* `stop_loss`\n' - '*Direction:* `Long`\n' - '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' - '*Remaining:* `(0.01 ETH, -24.812 USD)`' + old_convamount = telegram._rpc._fiat_converter.convert_amount + telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812 + telegram.send_msg({ + 'type': RPCMessageType.EXIT, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'leverage': 1.0, + 'direction': 'Long', + 'gain': 'loss', + 'order_rate': 3.201e-05, + 'amount': 1333.3333333333335, + 'order_type': 'market', + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_ratio': -0.57405275, + 'stake_currency': 'ETH', + 'fiat_currency': 'USD', + 'enter_tag': 'buy_signal1', + 'exit_reason': ExitType.STOP_LOSS.value, + 'open_date': arrow.utcnow().shift(hours=-1), + 'close_date': arrow.utcnow(), + }) + assert msg_mock.call_args[0][0] == ( + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Enter Tag:* `buy_signal1`\n' + '*Exit Reason:* `stop_loss`\n' + '*Direction:* `Long`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1:00:00 (60.0 min)`' ) - msg_mock.reset_mock() - telegram.send_msg({ - 'type': RPCMessageType.EXIT, - 'trade_id': 1, - 'exchange': 'Binance', - 'pair': 'KEY/ETH', - 'direction': 'Long', - 'gain': 'loss', - 'order_rate': 3.201e-05, - 'amount': 1333.3333333333335, - 'order_type': 'market', - 'open_rate': 7.5e-05, - 'current_rate': 3.201e-05, - 'profit_amount': -0.05746268, - 'profit_ratio': -0.57405275, - 'stake_currency': 'ETH', - 'enter_tag': 'buy_signal1', - 'exit_reason': ExitType.STOP_LOSS.value, - 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), - 'close_date': arrow.utcnow(), - }) - assert msg_mock.call_args[0][0] == ( - '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' - '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' - '*Enter Tag:* `buy_signal1`\n' - '*Exit Reason:* `stop_loss`\n' - '*Direction:* `Long`\n' - '*Amount:* `1333.33333333`\n' - '*Open Rate:* `0.00007500`\n' - '*Current Rate:* `0.00003201`\n' - '*Exit Rate:* `0.00003201`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`' - ) - # Reset singleton function to avoid random breaks - telegram._rpc._fiat_converter.convert_amount = old_convamount + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.EXIT, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'direction': 'Long', + 'gain': 'loss', + 'order_rate': 3.201e-05, + 'amount': 1333.3333333333335, + 'order_type': 'market', + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'cumulative_profit': -0.15746268, + 'profit_amount': -0.05746268, + 'profit_ratio': -0.57405275, + 'stake_currency': 'ETH', + 'fiat_currency': 'USD', + 'enter_tag': 'buy_signal1', + 'exit_reason': ExitType.STOP_LOSS.value, + 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'close_date': arrow.utcnow(), + 'stake_amount': 0.01, + 'sub_trade': True, + }) + assert msg_mock.call_args[0][0] == ( + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' + '*Unrealized Sub Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Cumulative Profit:* (`-0.15746268 ETH / -24.812 USD`)\n' + '*Enter Tag:* `buy_signal1`\n' + '*Exit Reason:* `stop_loss`\n' + '*Direction:* `Long`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Exit Rate:* `0.00003201`\n' + '*Remaining:* `(0.01 ETH, -24.812 USD)`' + ) + + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.EXIT, + 'trade_id': 1, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'direction': 'Long', + 'gain': 'loss', + 'order_rate': 3.201e-05, + 'amount': 1333.3333333333335, + 'order_type': 'market', + 'open_rate': 7.5e-05, + 'current_rate': 3.201e-05, + 'profit_amount': -0.05746268, + 'profit_ratio': -0.57405275, + 'stake_currency': 'ETH', + 'enter_tag': 'buy_signal1', + 'exit_reason': ExitType.STOP_LOSS.value, + 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), + 'close_date': arrow.utcnow(), + }) + assert msg_mock.call_args[0][0] == ( + '\N{WARNING SIGN} *Binance (dry):* Exiting KEY/ETH (#1)\n' + '*Unrealized Profit:* `-57.41% (loss: -0.05746268 ETH)`\n' + '*Enter Tag:* `buy_signal1`\n' + '*Exit Reason:* `stop_loss`\n' + '*Direction:* `Long`\n' + '*Amount:* `1333.33333333`\n' + '*Open Rate:* `0.00007500`\n' + '*Current Rate:* `0.00003201`\n' + '*Exit Rate:* `0.00003201`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`' + ) + # Reset singleton function to avoid random breaks + telegram._rpc._fiat_converter.convert_amount = old_convamount def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: @@ -2066,7 +2067,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker, direction, default_conf['telegram']['notification_settings']['exit_fill'] = 'on' telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) - with time_machine.travel("2022-09-01 05:00:00 +00:00", tick=False) as t: + with time_machine.travel("2022-09-01 05:00:00 +00:00", tick=False): telegram.send_msg({ 'type': RPCMessageType.EXIT_FILL, 'trade_id': 1, From 980a5a9b521d1a905a7beae383fd9ff8a8fd5302 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Dec 2022 19:54:04 +0100 Subject: [PATCH 22/23] Fix docs typo --- freqtrade/plugins/pairlist/VolumePairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index ad27a93d8..be58ec1a1 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -218,7 +218,7 @@ class VolumePairList(IPairList): else: filtered_tickers[i]['quoteVolume'] = 0 else: - # Tickers mode - filter based on incomming pairlist. + # Tickers mode - filter based on incoming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] if self._min_value > 0: From 6b9f3f279587e1097915732ad3ac6e69c00c9bb5 Mon Sep 17 00:00:00 2001 From: Emre Date: Sun, 11 Dec 2022 13:24:24 +0300 Subject: [PATCH 23/23] Fix test validation --- freqtrade/configuration/config_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 7e291cb90..606f081ef 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -360,7 +360,7 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None: feature_parameters = conf.get('freqai', {}).get('feature_parameters', {}) include_timeframes = [main_tf] + freqai_include_timeframes conf.get('freqai', {}).get('feature_parameters', {}) \ - .update({'include_timeframes': include_timeframes, **feature_parameters}) + .update({**feature_parameters, 'include_timeframes': include_timeframes}) def _validate_freqai_backtest(conf: Dict[str, Any]) -> None: