From 25724ef7296be44905406f96815c2e41ce69d815 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 16:02:36 +0100 Subject: [PATCH 01/57] Version bump 2023.2 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f531bb605..f25bb2e52 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.1' +__version__ = '2023.2' if 'dev' in __version__: from pathlib import Path From 2ea77b22e0386e3f2d9a78025895b5949795f267 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Mar 2023 06:56:49 +0200 Subject: [PATCH 02/57] Bump version to 2023.3 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f25bb2e52..f56328674 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.2' +__version__ = '2023.3' if 'dev' in __version__: from pathlib import Path From d40a631565fd3b1907ed7035213b2d5c69107bed Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 Apr 2023 13:53:27 +0200 Subject: [PATCH 03/57] Version bump 2023.4 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f56328674..f190e7204 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.3' +__version__ = '2023.4' if 'dev' in __version__: from pathlib import Path From 4eb427533158eaffe9b71606995eee08e4a76817 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 09:53:46 +0200 Subject: [PATCH 04/57] Fix volatilityfilter behavior closes #8698 --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 9196026bb..baf4fcd26 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -74,7 +74,7 @@ class VolatilityFilter(IPairList): needed_pairs: ListPairsWithTimeframes = [ (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] - since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days - 1)) + since_ms = dt_ts(dt_floor_day(dt_now()) - timedelta(days=self._days)) # Get all candles candles = {} if needed_pairs: From a3473f3f60b288fefec42d546d05347d09f82a43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 09:59:57 +0200 Subject: [PATCH 05/57] Better handling of shift --- freqtrade/plugins/pairlist/VolatilityFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index baf4fcd26..61a1dcbf0 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -103,7 +103,7 @@ class VolatilityFilter(IPairList): result = False if daily_candles is not None and not daily_candles.empty: - returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) + returns = (np.log(daily_candles["close"].shift(1) / daily_candles["close"])) returns.fillna(0, inplace=True) volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days) From c9f78afe65fc227a3c3f59b04981dbd8afa902cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 May 2023 20:02:03 +0200 Subject: [PATCH 06/57] Bump version to 2023.5 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f190e7204..e7f01f9a5 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.4' +__version__ = '2023.5' if 'dev' in __version__: from pathlib import Path From beaaa94406a120ee42c317ed64f26fe3d9ccdb35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 11:46:31 +0200 Subject: [PATCH 07/57] Improve test for reload-markets timings, fix bug closes #8714 --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 3b1466c69..27e6ea63a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -191,7 +191,7 @@ class Exchange: # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_conf.get( - "markets_refresh_interval", 60) * 60 + "markets_refresh_interval", 60) * 60 * 1000 if self.trading_mode != TradingMode.SPOT and load_leverage_tiers: self.fill_leverage_tiers() diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ef70c8ba1..f022a0905 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -633,21 +633,23 @@ def test__load_markets(default_conf, mocker, caplog): assert ex.markets == expected_return -def test_reload_markets(default_conf, mocker, caplog): +def test_reload_markets(default_conf, mocker, caplog, time_machine): caplog.set_level(logging.DEBUG) initial_markets = {'ETH/BTC': {}} updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}} - + start_dt = dt_now() + time_machine.move_to(start_dt, tick=False) api_mock = MagicMock() api_mock.load_markets = MagicMock(return_value=initial_markets) default_conf['exchange']['markets_refresh_interval'] = 10 exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) exchange._load_async_markets = MagicMock() - exchange._last_markets_refresh = dt_ts() + assert exchange._last_markets_refresh == dt_ts() assert exchange.markets == initial_markets + time_machine.move_to(start_dt + timedelta(minutes=8), tick=False) # less than 10 minutes have passed, no reload exchange.reload_markets() assert exchange.markets == initial_markets @@ -655,12 +657,18 @@ def test_reload_markets(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value=updated_markets) # more than 10 minutes have passed, reload is executed - exchange._last_markets_refresh = dt_ts(dt_now() - timedelta(minutes=15)) + time_machine.move_to(start_dt + timedelta(minutes=11), tick=False) exchange.reload_markets() assert exchange.markets == updated_markets assert exchange._load_async_markets.call_count == 1 assert log_has('Performing scheduled market reload..', caplog) + # Not called again + exchange._load_async_markets.reset_mock() + + exchange.reload_markets() + assert exchange._load_async_markets.call_count == 0 + def test_reload_markets_exception(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) From 61f1701e561c7ffc07104de4d82a7407b90ab31e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Jun 2023 22:02:33 +0200 Subject: [PATCH 08/57] Bump version to 2023.5.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e7f01f9a5..bd54de301 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.5' +__version__ = '2023.5.1' if 'dev' in __version__: from pathlib import Path From 859f7ff3de9a3101b7cb3d016bd5c18cc95bf930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Jun 2023 18:29:37 +0200 Subject: [PATCH 09/57] be explicit when loading pairs file. --- freqtrade/configuration/configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index a64eaa0ca..43ede568c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -568,6 +568,7 @@ class Configuration: # Fall back to /dl_path/pairs.json pairs_file = config['datadir'] / 'pairs.json' if pairs_file.exists(): + logger.info(f'Reading pairs file "{pairs_file}".') config['pairs'] = load_file(pairs_file) if 'pairs' in config and isinstance(config['pairs'], list): config['pairs'].sort() From 8c54036fa539a577627cb0059f2e4ca4addc296b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 06:45:56 +0200 Subject: [PATCH 10/57] Move Downloading tip from pairs file section --- docs/data-download.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index a7b1987aa..4ee7aba02 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -83,6 +83,11 @@ Common arguments: ``` +!!! Tip "Downloading all data for one quote currency" + Often, you'll want to download data for all pairs of a specific quote-currency. In such cases, you can use the following shorthand: + `freqtrade download-data --exchange binance --pairs .*/USDT <...>`. The provided "pairs" string will be expanded to contain all active pairs on the exchange. + To also download data for inactive (delisted) pairs, add `--include-inactive-pairs` to the command. + !!! Note "Startup period" `download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored. @@ -113,11 +118,6 @@ Mixing different stake-currencies is allowed for this file, since it's only used ] ``` -!!! Tip "Downloading all data for one quote currency" - Often, you'll want to download data for all pairs of a specific quote-currency. In such cases, you can use the following shorthand: - `freqtrade download-data --exchange binance --pairs .*/USDT <...>`. The provided "pairs" string will be expanded to contain all active pairs on the exchange. - To also download data for inactive (delisted) pairs, add `--include-inactive-pairs` to the command. - ??? Note "Permission denied errors" If your configuration directory `user_data` was made by docker, you may get the following error: From b0e5fb3940ee267f721df046b2194545ad6f2935 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 06:50:59 +0200 Subject: [PATCH 11/57] Improve structure of download-data documentation --- docs/data-download.md | 94 +++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 4ee7aba02..1588d6eb5 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -93,44 +93,6 @@ Common arguments: For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period). -### Pairs file - -In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. -If you are using Binance for example: - -- create a directory `user_data/data/binance` and copy or create the `pairs.json` file in that directory. -- update the `pairs.json` file to contain the currency pairs you are interested in. - -```bash -mkdir -p user_data/data/binance -touch user_data/data/binance/pairs.json -``` - -The format of the `pairs.json` file is a simple json list. -Mixing different stake-currencies is allowed for this file, since it's only used for downloading. - -``` json -[ - "ETH/BTC", - "ETH/USDT", - "BTC/USDT", - "XRP/ETH" -] -``` - -??? Note "Permission denied errors" - If your configuration directory `user_data` was made by docker, you may get the following error: - - ``` - cp: cannot create regular file 'user_data/data/binance/pairs.json': Permission denied - ``` - - You can fix the permissions of your user-data directory as follows: - - ``` - sudo chown -R $UID:$GID user_data - ``` - ### Start download Then run: @@ -163,6 +125,19 @@ freqtrade download-data --exchange binance --pairs .*/USDT - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +??? Note "Permission denied errors" + If your configuration directory `user_data` was made by docker, you may get the following error: + + ``` + cp: cannot create regular file 'user_data/data/binance/pairs.json': Permission denied + ``` + + You can fix the permissions of your user-data directory as follows: + + ``` + sudo chown -R $UID:$GID user_data + ``` + #### Download additional data before the current timerange Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. @@ -238,7 +213,32 @@ Size has been taken from the BTC/USDT 1m spot combination for the timerange spec To have a best performance/size mix, we recommend the use of either feather or parquet. -#### Sub-command convert data +### Pairs file + +In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. +If you are using Binance for example: + +- create a directory `user_data/data/binance` and copy or create the `pairs.json` file in that directory. +- update the `pairs.json` file to contain the currency pairs you are interested in. + +```bash +mkdir -p user_data/data/binance +touch user_data/data/binance/pairs.json +``` + +The format of the `pairs.json` file is a simple json list. +Mixing different stake-currencies is allowed for this file, since it's only used for downloading. + +``` json +[ + "ETH/BTC", + "ETH/USDT", + "BTC/USDT", + "XRP/ETH" +] +``` + +## Sub-command convert data ``` usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -290,7 +290,7 @@ Common arguments: ``` -##### Example converting data +### Example converting data The following command will convert all candle (OHLCV) data available in `~/.freqtrade/data/binance` from json to jsongz, saving diskspace in the process. It'll also remove original json data files (`--erase` parameter). @@ -299,7 +299,7 @@ It'll also remove original json data files (`--erase` parameter). freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase ``` -#### Sub-command convert trade data +## Sub-command convert trade data ``` usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -342,7 +342,7 @@ Common arguments: ``` -##### Example converting trades +### Example converting trades The following command will convert all available trade-data in `~/.freqtrade/data/kraken` from jsongz to json. It'll also remove original jsongz data files (`--erase` parameter). @@ -351,7 +351,7 @@ It'll also remove original jsongz data files (`--erase` parameter). freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase ``` -### Sub-command trades to ohlcv +## Sub-command trades to ohlcv When you need to use `--dl-trades` (kraken only) to download data, conversion of trades data to ohlcv data is the last step. This command will allow you to repeat this last step for additional timeframes without re-downloading the data. @@ -400,13 +400,13 @@ Common arguments: ``` -#### Example trade-to-ohlcv conversion +### Example trade-to-ohlcv conversion ``` bash freqtrade trades-to-ohlcv --exchange kraken -t 5m 1h 1d --pairs BTC/EUR ETH/EUR ``` -### Sub-command list-data +## Sub-command list-data You can get a list of downloaded data using the `list-data` sub-command. @@ -451,7 +451,7 @@ Common arguments: ``` -#### Example list-data +### Example list-data ```bash > freqtrade list-data --userdir ~/.freqtrade/user_data/ @@ -465,7 +465,7 @@ ETH/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d ETH/USDT 5m, 15m, 30m, 1h, 2h, 4h ``` -### Trades (tick) data +## Trades (tick) data By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API. This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes. From 96c2ca67e9010242d93b2bc5a5ed7d37272fee73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 06:51:11 +0200 Subject: [PATCH 12/57] Add usage note for pairs.json file --- docs/data-download.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/data-download.md b/docs/data-download.md index 1588d6eb5..06ef1a355 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -238,6 +238,10 @@ Mixing different stake-currencies is allowed for this file, since it's only used ] ``` +!!! Note + The `pairs.json` file is only used when no configuration is loaded (implicitly by naming, or via `--config` flag). + You can force the usage of this file via `--pairs-file pairs.json` - however we recommend to use the pairlist from within the configuration, either via `exchange.pair_whitelist` or `pairs` setting in the configuration. + ## Sub-command convert data ``` From 5d60c626454120ed8fdb196342d5e3dae5b3a3ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 06:55:19 +0200 Subject: [PATCH 13/57] align list blocks --- docs/data-download.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 06ef1a355..7b63e4556 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -117,13 +117,13 @@ freqtrade download-data --exchange binance --pairs .*/USDT ### Other Notes -- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) -- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). -- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. -- Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. -- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +* To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. +* To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) +* To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. +* To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). +* To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. +* Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. +* To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. ??? Note "Permission denied errors" If your configuration directory `user_data` was made by docker, you may get the following error: @@ -138,7 +138,7 @@ freqtrade download-data --exchange binance --pairs .*/USDT sudo chown -R $UID:$GID user_data ``` -#### Download additional data before the current timerange +### Download additional data before the current timerange Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. You can do so by using the `--prepend` flag, combined with `--timerange` - specifying an end-date. @@ -218,8 +218,8 @@ To have a best performance/size mix, we recommend the use of either feather or p In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. If you are using Binance for example: -- create a directory `user_data/data/binance` and copy or create the `pairs.json` file in that directory. -- update the `pairs.json` file to contain the currency pairs you are interested in. +* create a directory `user_data/data/binance` and copy or create the `pairs.json` file in that directory. +* update the `pairs.json` file to contain the currency pairs you are interested in. ```bash mkdir -p user_data/data/binance From c7683a7b618850971e51dab8764b392eb2a1a718 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 06:57:48 +0200 Subject: [PATCH 14/57] Improve docs wording --- docs/data-download.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 7b63e4556..d45c7ef63 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -6,7 +6,7 @@ To download data (candles / OHLCV) needed for backtesting and hyperoptimization If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes for the last 30 days. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). -Otherwise `--exchange` becomes mandatory. +Without provided configuration, `--exchange` becomes mandatory. You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. @@ -95,13 +95,13 @@ Common arguments: ### Start download -Then run: +A very simple command (assuming an available `config.json` file) can look as follows. ```bash freqtrade download-data --exchange binance ``` -This will download historical candle (OHLCV) data for all the currency pairs you defined in `pairs.json`. +This will download historical candle (OHLCV) data for all the currency pairs defined in the configuration. Alternatively, specify the pairs directly @@ -109,7 +109,7 @@ Alternatively, specify the pairs directly freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT ``` -or as regex (to download all active USDT pairs) +or as regex (in this case, to download all active USDT pairs) ```bash freqtrade download-data --exchange binance --pairs .*/USDT From b49a11876437db303c126b384492c712698faaf5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 18:14:16 +0200 Subject: [PATCH 15/57] Fix exit_timeout test --- tests/test_freqtradebot.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 24e726403..7da6c32c2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2976,11 +2976,12 @@ def test_manage_open_orders_exit_usercustom( ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} open_trade_usdt.open_order_id = limit_sell_order_old['id'] - order = Order.parse_from_ccxt_object(limit_sell_order_old, 'mocked', 'sell') - open_trade_usdt.orders[0] = order if is_short: limit_sell_order_old['side'] = 'buy' open_trade_usdt.is_short = is_short + open_exit_order = Order.parse_from_ccxt_object(limit_sell_order_old, 'mocked', + 'buy' if is_short else 'sell') + open_trade_usdt.orders[-1] = open_exit_order rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() @@ -3011,8 +3012,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) - assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) + assert freqtrade.strategy.check_exit_timeout.call_count == 1 + assert freqtrade.strategy.check_entry_timeout.call_count == 0 freqtrade.strategy.check_exit_timeout = MagicMock(side_effect=KeyError) freqtrade.strategy.check_entry_timeout = MagicMock(side_effect=KeyError) @@ -3020,8 +3021,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) - assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) + assert freqtrade.strategy.check_exit_timeout.call_count == 1 + assert freqtrade.strategy.check_entry_timeout.call_count == 0 # Return True - sells! freqtrade.strategy.check_exit_timeout = MagicMock(return_value=True) @@ -3029,8 +3030,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 - assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) - assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) + assert freqtrade.strategy.check_exit_timeout.call_count == 1 + assert freqtrade.strategy.check_entry_timeout.call_count == 0 trade = Trade.session.scalars(select(Trade)).first() # cancelling didn't succeed - order-id remains open. assert trade.open_order_id is not None From 2f7b29ed34ad868264ac56945445cb08517203a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 19:08:55 +0200 Subject: [PATCH 16/57] Fix test_tsl_on_exchange_compatible_with_edge --- tests/test_freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 7da6c32c2..2fc670246 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1992,7 +1992,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde enter_order = limit_order['buy'] exit_order = limit_order['sell'] - + enter_order['average'] = 2.19 # When trailing stoploss is set stoploss = MagicMock(return_value={'id': '13434334', 'status': 'open'}) patch_RPCManager(mocker) @@ -2009,8 +2009,8 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde 'last': 2.19 }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, create_stoploss=stoploss, @@ -2106,7 +2106,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde assert trade.stop_loss == 4.4 * 0.99 cancel_order_mock.assert_called_once_with('100', 'NEO/BTC') stoploss_order_mock.assert_called_once_with( - amount=pytest.approx(11.41438356), + amount=30, pair='NEO/BTC', order_types=freqtrade.strategy.order_types, stop_price=4.4 * 0.99, From 01dfca80aba1de1e54009af9f4b0b9745f44feff Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Jun 2023 19:16:21 +0200 Subject: [PATCH 17/57] Improve stop test behavior --- tests/test_freqtradebot.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2fc670246..3bfa5a127 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1605,13 +1605,13 @@ def test_create_stoploss_order_insufficient_funds( assert mock_insuf.call_count == 1 -@pytest.mark.parametrize("is_short,bid,ask,stop_price,amt,hang_price", [ - (False, [4.38, 4.16], [4.4, 4.17], ['2.0805', 4.4 * 0.95], 27.39726027, 3), - (True, [1.09, 1.21], [1.1, 1.22], ['2.321', 1.09 * 1.05], 27.27272727, 1.5), +@pytest.mark.parametrize("is_short,bid,ask,stop_price,hang_price", [ + (False, [4.38, 4.16], [4.4, 4.17], ['2.0805', 4.4 * 0.95], 3), + (True, [1.09, 1.21], [1.1, 1.22], ['2.321', 1.09 * 1.05], 1.5), ]) @pytest.mark.usefixtures("init_persistence") def test_handle_stoploss_on_exchange_trailing( - mocker, default_conf_usdt, fee, is_short, bid, ask, limit_order, stop_price, amt, hang_price + mocker, default_conf_usdt, fee, is_short, bid, ask, limit_order, stop_price, hang_price ) -> None: # When trailing stoploss is set enter_order = limit_order[entry_side(is_short)] @@ -1626,8 +1626,8 @@ def test_handle_stoploss_on_exchange_trailing( 'last': 2.19, }), create_order=MagicMock(side_effect=[ - {'id': enter_order['id']}, - {'id': exit_order['id']}, + enter_order, + exit_order, ]), get_fee=fee, ) @@ -1723,7 +1723,7 @@ def test_handle_stoploss_on_exchange_trailing( cancel_order_mock.assert_called_once_with('100', 'ETH/USDT') stoploss_order_mock.assert_called_once_with( - amount=pytest.approx(amt), + amount=30, pair='ETH/USDT', order_types=freqtrade.strategy.order_types, stop_price=stop_price[1], From 757c6dc5cacec9fbc3074a51bda47f8e00b14b6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 23 Jun 2023 18:15:06 +0200 Subject: [PATCH 18/57] Use Self typing --- freqtrade/configuration/timerange.py | 10 ++++++---- freqtrade/persistence/trade_model.py | 15 ++++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index cff35db7e..23e3a6b60 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -6,6 +6,8 @@ import re from datetime import datetime, timezone from typing import Optional +from typing_extensions import Self + from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.exceptions import OperationalException @@ -107,15 +109,15 @@ class TimeRange: self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles) self.starttype = 'date' - @staticmethod - def parse_timerange(text: Optional[str]) -> 'TimeRange': + @classmethod + def parse_timerange(cls, text: Optional[str]) -> Self: """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange :return: Start and End range period """ if not text: - return TimeRange(None, None, 0, 0) + return cls(None, None, 0, 0) syntax = [(r'^-(\d{8})$', (None, 'date')), (r'^(\d{8})-$', ('date', None)), (r'^(\d{8})-(\d{8})$', ('date', 'date')), @@ -156,5 +158,5 @@ class TimeRange: if start > stop > 0: raise OperationalException( f'Start date is after stop date for timerange "{text}"') - return TimeRange(stype[0], stype[1], start, stop) + return cls(stype[0], stype[1], start, stop) raise OperationalException(f'Incorrect syntax for timerange "{text}"') diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ddc147763..4d3564df6 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -10,6 +10,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Sequence, cast from sqlalchemy import (Enum, Float, ForeignKey, Integer, ScalarResult, Select, String, UniqueConstraint, desc, func, select) from sqlalchemy.orm import Mapped, lazyload, mapped_column, relationship, validates +from typing_extensions import Self from freqtrade.constants import (CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) @@ -246,15 +247,15 @@ class Order(ModelBase): else: logger.warning(f"Did not find order for {order}.") - @staticmethod + @classmethod def parse_from_ccxt_object( - order: Dict[str, Any], pair: str, side: str, - amount: Optional[float] = None, price: Optional[float] = None) -> 'Order': + cls, order: Dict[str, Any], pair: str, side: str, + amount: Optional[float] = None, price: Optional[float] = None) -> Self: """ Parse an order from a ccxt object and return a new order Object. Optional support for overriding amount and price is only used for test simplification. """ - o = Order( + o = cls( order_id=str(order['id']), ft_order_side=side, ft_pair=pair, @@ -1641,8 +1642,8 @@ class Trade(ModelBase, LocalTrade): )).scalar_one() return trading_volume - @staticmethod - def from_json(json_str: str) -> 'Trade': + @classmethod + def from_json(cls, json_str: str) -> Self: """ Create a Trade instance from a json string. @@ -1652,7 +1653,7 @@ class Trade(ModelBase, LocalTrade): """ import rapidjson data = rapidjson.loads(json_str) - trade = Trade( + trade = cls( id=data["trade_id"], pair=data["pair"], base_currency=data["base_currency"], From 69087c30e738d1e2a31233221e375fb4a35871f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Jun 2023 20:18:24 +0200 Subject: [PATCH 19/57] Don't overwrite "type" with a variable --- freqtrade/rpc/api_server/api_ws.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/api_server/api_ws.py b/freqtrade/rpc/api_server/api_ws.py index b253d66c2..1a2f528f5 100644 --- a/freqtrade/rpc/api_server/api_ws.py +++ b/freqtrade/rpc/api_server/api_ws.py @@ -62,13 +62,13 @@ async def _process_consumer_request( logger.error(f"Invalid request from {channel}: {e}") return - type, data = websocket_request.type, websocket_request.data + type_, data = websocket_request.type, websocket_request.data response: WSMessageSchema - logger.debug(f"Request of type {type} from {channel}") + logger.debug(f"Request of type {type_} from {channel}") # If we have a request of type SUBSCRIBE, set the topics in this channel - if type == RPCRequestType.SUBSCRIBE: + if type_ == RPCRequestType.SUBSCRIBE: # If the request is empty, do nothing if not data: return @@ -80,7 +80,7 @@ async def _process_consumer_request( # We don't send a response for subscriptions return - elif type == RPCRequestType.WHITELIST: + elif type_ == RPCRequestType.WHITELIST: # Get whitelist whitelist = rpc._ws_request_whitelist() @@ -88,7 +88,7 @@ async def _process_consumer_request( response = WSWhitelistMessage(data=whitelist) await channel.send(response.dict(exclude_none=True)) - elif type == RPCRequestType.ANALYZED_DF: + elif type_ == RPCRequestType.ANALYZED_DF: # Limit the amount of candles per dataframe to 'limit' or 1500 limit = int(min(data.get('limit', 1500), 1500)) if data else None pair = data.get('pair', None) if data else None From 5f98530ef93b801ec2536b980cfac3a5dfaf28c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Jun 2023 20:20:54 +0200 Subject: [PATCH 20/57] Catch and send exceptions from websockets --- freqtrade/rpc/api_server/api_ws.py | 14 +++++++++++--- freqtrade/rpc/api_server/ws_schemas.py | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_ws.py b/freqtrade/rpc/api_server/api_ws.py index 1a2f528f5..40a5a75fd 100644 --- a/freqtrade/rpc/api_server/api_ws.py +++ b/freqtrade/rpc/api_server/api_ws.py @@ -7,12 +7,14 @@ from fastapi.websockets import WebSocket from pydantic import ValidationError from freqtrade.enums import RPCMessageType, RPCRequestType +from freqtrade.exceptions import FreqtradeException from freqtrade.rpc.api_server.api_auth import validate_ws_token from freqtrade.rpc.api_server.deps import get_message_stream, get_rpc from freqtrade.rpc.api_server.ws.channel import WebSocketChannel, create_channel from freqtrade.rpc.api_server.ws.message_stream import MessageStream -from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSMessageSchema, - WSRequestSchema, WSWhitelistMessage) +from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSErrorMessage, + WSMessageSchema, WSRequestSchema, + WSWhitelistMessage) from freqtrade.rpc.rpc import RPC @@ -27,7 +29,13 @@ async def channel_reader(channel: WebSocketChannel, rpc: RPC): Iterate over the messages from the channel and process the request """ async for message in channel: - await _process_consumer_request(message, channel, rpc) + try: + await _process_consumer_request(message, channel, rpc) + except FreqtradeException: + logger.exception(f"Error processing request from {channel}") + response = WSErrorMessage(data='Error processing request') + + await channel.send(response.dict(exclude_none=True)) async def channel_broadcaster(channel: WebSocketChannel, message_stream: MessageStream): diff --git a/freqtrade/rpc/api_server/ws_schemas.py b/freqtrade/rpc/api_server/ws_schemas.py index 292672b60..af98bd532 100644 --- a/freqtrade/rpc/api_server/ws_schemas.py +++ b/freqtrade/rpc/api_server/ws_schemas.py @@ -66,4 +66,9 @@ class WSAnalyzedDFMessage(WSMessageSchema): type: RPCMessageType = RPCMessageType.ANALYZED_DF data: AnalyzedDFData + +class WSErrorMessage(WSMessageSchema): + type: RPCMessageType = RPCMessageType.EXCEPTION + data: str + # -------------------------------------------------------------------------- From 48e896532204fad08b590ab7ba0b4cd14398eedb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 15:35:57 +0200 Subject: [PATCH 21/57] Don't add header if it's not needed --- freqtrade/commands/list_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index dcb102ce5..84f237f77 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -33,11 +33,11 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: else: headers = { 'name': 'Exchange name', - 'valid': 'Valid', 'supported': 'Supported', 'trade_modes': 'Markets', 'comment': 'Reason', } + headers.update({'valid': 'Valid'} if args['list_exchanges_all'] else {}) def build_entry(exchange: ValidExchangesType, valid: bool): valid_entry = {'valid': exchange['valid']} if valid else {} From fd420738cd2342702530a12d12885193064e3553 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 25 Jun 2023 15:43:02 +0200 Subject: [PATCH 22/57] ensure outlier-check is returning as a numpy array from datasieve --- docs/freqai-configuration.md | 2 +- docs/strategy_migration.md | 2 +- freqtrade/freqai/base_models/BaseClassifierModel.py | 2 +- freqtrade/freqai/base_models/BasePyTorchClassifier.py | 2 +- freqtrade/freqai/base_models/BasePyTorchRegressor.py | 2 +- freqtrade/freqai/base_models/BaseRegressionModel.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- .../freqai/prediction_models/PyTorchTransformerRegressor.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 6e2ed8379..090fa8415 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -160,7 +160,7 @@ Below are the values you can expect to include/use inside a typical strategy dat |------------|-------------| | `df['&*']` | Any dataframe column prepended with `&` in `set_freqai_targets()` is treated as a training target (label) inside FreqAI (typically following the naming convention `&-s*`). For example, to predict the close price 40 candles into the future, you would set `df['&-s_close'] = df['close'].shift(-self.freqai_info["feature_parameters"]["label_period_candles"])` with `"label_period_candles": 40` in the config. FreqAI makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`.
**Datatype:** Depends on the output of the model. | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`).
**Datatype:** Float. -| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`.
**Datatype:** Integer between -2 and 2. +| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`.
**Datatype:** Integer between -2 and 2. | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di).
**Datatype:** Float. | `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md).
**Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`.
**Datatype:** Depends on the output of the model. diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index 353da0ccb..d32d5880c 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -801,7 +801,7 @@ class MyCoolFreqaiModel(BaseRegressionModel): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers # ... your custom code return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BaseClassifierModel.py b/freqtrade/freqai/base_models/BaseClassifierModel.py index f35b07e66..810fb6894 100644 --- a/freqtrade/freqai/base_models/BaseClassifierModel.py +++ b/freqtrade/freqai/base_models/BaseClassifierModel.py @@ -121,6 +121,6 @@ class BaseClassifierModel(IFreqaiModel): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BasePyTorchClassifier.py b/freqtrade/freqai/base_models/BasePyTorchClassifier.py index c47c5069a..040549ac1 100644 --- a/freqtrade/freqai/base_models/BasePyTorchClassifier.py +++ b/freqtrade/freqai/base_models/BasePyTorchClassifier.py @@ -95,7 +95,7 @@ class BasePyTorchClassifier(BasePyTorchModel): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BasePyTorchRegressor.py b/freqtrade/freqai/base_models/BasePyTorchRegressor.py index 325743134..71d384653 100644 --- a/freqtrade/freqai/base_models/BasePyTorchRegressor.py +++ b/freqtrade/freqai/base_models/BasePyTorchRegressor.py @@ -56,7 +56,7 @@ class BasePyTorchRegressor(BasePyTorchModel): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers return (pred_df, dk.do_predict) def train( diff --git a/freqtrade/freqai/base_models/BaseRegressionModel.py b/freqtrade/freqai/base_models/BaseRegressionModel.py index 2e07d3fb7..6f83756dc 100644 --- a/freqtrade/freqai/base_models/BaseRegressionModel.py +++ b/freqtrade/freqai/base_models/BaseRegressionModel.py @@ -115,6 +115,6 @@ class BaseRegressionModel(IFreqaiModel): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 4ca5467b6..b053e276f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1013,5 +1013,5 @@ class IFreqaiModel(ABC): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers return diff --git a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py index bf78488ff..857c1edb8 100644 --- a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py @@ -137,7 +137,7 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor): dk.DI_values = dk.feature_pipeline["di"].di_values else: dk.DI_values = np.zeros(len(outliers.index)) - dk.do_predict = outliers.to_numpy() + dk.do_predict = outliers if x.shape[1] > 1: zeros_df = pd.DataFrame(np.zeros((x.shape[1] - len(pred_df), len(pred_df.columns))), From 9da28e53288e9010bb99d172817fa5d3275096ba Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 25 Jun 2023 15:44:24 +0200 Subject: [PATCH 23/57] bump datasieve --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 2eacbaffb..e8e3b9334 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -9,4 +9,4 @@ catboost==1.2; 'arm' not in platform_machine lightgbm==3.3.5 xgboost==1.7.6 tensorboard==2.13.0 -datasieve==0.1.5 +datasieve==0.1.6 From fca73531cfae5da40cf706fcd34afc596ed533ba Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 25 Jun 2023 16:34:44 +0200 Subject: [PATCH 24/57] fix: use .shape instead of index for outliers --- docs/strategy_migration.md | 2 +- freqtrade/freqai/base_models/BaseClassifierModel.py | 2 +- freqtrade/freqai/base_models/BasePyTorchClassifier.py | 2 +- freqtrade/freqai/base_models/BasePyTorchRegressor.py | 2 +- freqtrade/freqai/base_models/BaseRegressionModel.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- .../freqai/prediction_models/PyTorchTransformerRegressor.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index d32d5880c..d00349d1d 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -800,7 +800,7 @@ class MyCoolFreqaiModel(BaseRegressionModel): if self.freqai_info.get("DI_threshold", 0) > 0: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers # ... your custom code diff --git a/freqtrade/freqai/base_models/BaseClassifierModel.py b/freqtrade/freqai/base_models/BaseClassifierModel.py index 810fb6894..42b5c1a0e 100644 --- a/freqtrade/freqai/base_models/BaseClassifierModel.py +++ b/freqtrade/freqai/base_models/BaseClassifierModel.py @@ -120,7 +120,7 @@ class BaseClassifierModel(IFreqaiModel): if dk.feature_pipeline["di"]: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BasePyTorchClassifier.py b/freqtrade/freqai/base_models/BasePyTorchClassifier.py index 040549ac1..4780af818 100644 --- a/freqtrade/freqai/base_models/BasePyTorchClassifier.py +++ b/freqtrade/freqai/base_models/BasePyTorchClassifier.py @@ -94,7 +94,7 @@ class BasePyTorchClassifier(BasePyTorchModel): if dk.feature_pipeline["di"]: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BasePyTorchRegressor.py b/freqtrade/freqai/base_models/BasePyTorchRegressor.py index 71d384653..83fea4ef9 100644 --- a/freqtrade/freqai/base_models/BasePyTorchRegressor.py +++ b/freqtrade/freqai/base_models/BasePyTorchRegressor.py @@ -55,7 +55,7 @@ class BasePyTorchRegressor(BasePyTorchModel): if dk.feature_pipeline["di"]: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/base_models/BaseRegressionModel.py b/freqtrade/freqai/base_models/BaseRegressionModel.py index 6f83756dc..179e4be87 100644 --- a/freqtrade/freqai/base_models/BaseRegressionModel.py +++ b/freqtrade/freqai/base_models/BaseRegressionModel.py @@ -114,7 +114,7 @@ class BaseRegressionModel(IFreqaiModel): if dk.feature_pipeline["di"]: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers return (pred_df, dk.do_predict) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index b053e276f..9fe8bd194 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1012,6 +1012,6 @@ class IFreqaiModel(ABC): if self.freqai_info.get("DI_threshold", 0) > 0: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers return diff --git a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py index 857c1edb8..a76bab05c 100644 --- a/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py +++ b/freqtrade/freqai/prediction_models/PyTorchTransformerRegressor.py @@ -136,7 +136,7 @@ class PyTorchTransformerRegressor(BasePyTorchRegressor): if self.freqai_info.get("DI_threshold", 0) > 0: dk.DI_values = dk.feature_pipeline["di"].di_values else: - dk.DI_values = np.zeros(len(outliers.index)) + dk.DI_values = np.zeros(outliers.shape[0]) dk.do_predict = outliers if x.shape[1] > 1: From 5e084ad2e5701eea5d007643d48926ab3ed0e207 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 16:59:53 +0200 Subject: [PATCH 25/57] convert optimize_reports to a package --- freqtrade/optimize/optimize_reports/__init__.py | 10 ++++++++++ .../{ => optimize_reports}/optimize_reports.py | 0 tests/optimize/test_backtesting.py | 2 +- tests/optimize/test_optimize_reports.py | 12 +++++++----- 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 freqtrade/optimize/optimize_reports/__init__.py rename freqtrade/optimize/{ => optimize_reports}/optimize_reports.py (100%) diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py new file mode 100644 index 000000000..df42e03bb --- /dev/null +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -0,0 +1,10 @@ +# flake8: noqa: F401 +from freqtrade.optimize.optimize_reports.optimize_reports import ( + generate_all_periodic_breakdown_stats, generate_backtest_stats, generate_daily_stats, + generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, + generate_periodic_breakdown_stats, generate_rejected_signals, generate_strategy_comparison, + generate_strategy_stats, generate_tag_metrics, generate_trade_signal_candles, + generate_trading_stats, generate_wins_draws_losses, show_backtest_result, show_backtest_results, + show_sorted_pairlist, store_backtest_analysis_results, store_backtest_stats, + text_table_add_metrics, text_table_bt_results, text_table_exit_reason, + text_table_periodic_breakdown, text_table_strategy, text_table_tags) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py similarity index 100% rename from freqtrade/optimize/optimize_reports.py rename to freqtrade/optimize/optimize_reports/optimize_reports.py diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index bef942b43..15be6bb28 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1437,7 +1437,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): strattable_mock = MagicMock() strat_summary = MagicMock() - mocker.patch.multiple('freqtrade.optimize.optimize_reports', + mocker.patch.multiple('freqtrade.optimize.optimize_reports.optimize_reports', text_table_bt_results=text_table_mock, text_table_strategy=strattable_mock, generate_pair_metrics=MagicMock(), diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 82e8a46fb..1ea2a7380 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -14,15 +14,16 @@ from freqtrade.data.btanalysis import (get_latest_backtest_filename, load_backte load_backtest_stats) from freqtrade.edge import PairInfo from freqtrade.enums import ExitType -from freqtrade.optimize.optimize_reports import (_get_resample_from_period, generate_backtest_stats, - generate_daily_stats, generate_edge_table, - generate_exit_reason_stats, generate_pair_metrics, +from freqtrade.optimize.optimize_reports import (generate_backtest_stats, generate_daily_stats, + generate_edge_table, generate_exit_reason_stats, + generate_pair_metrics, generate_periodic_breakdown_stats, generate_strategy_comparison, generate_trading_stats, show_sorted_pairlist, store_backtest_analysis_results, store_backtest_stats, text_table_bt_results, text_table_exit_reason, text_table_strategy) +from freqtrade.optimize.optimize_reports.optimize_reports import _get_resample_from_period from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.util import dt_ts from freqtrade.util.datetime_helpers import dt_from_ts, dt_utc @@ -209,7 +210,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): def test_store_backtest_stats(testdatadir, mocker): - dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json') + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.optimize_reports.file_dump_json') store_backtest_stats(testdatadir, {'metadata': {}}, '2022_01_01_15_05_13') @@ -228,7 +229,8 @@ def test_store_backtest_stats(testdatadir, mocker): def test_store_backtest_candles(testdatadir, mocker): - dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_joblib') + dump_mock = mocker.patch( + 'freqtrade.optimize.optimize_reports.optimize_reports.file_dump_joblib') candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} From 794bca13795f33731c49ba2d0bb58d573fb91ce2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 17:09:57 +0200 Subject: [PATCH 26/57] Split optimize report generation from visualization --- .../optimize/optimize_reports/__init__.py | 14 +- .../optimize_reports/optimize_reports.py | 374 +---------------- .../optimize_reports/visualization.py | 380 ++++++++++++++++++ tests/optimize/test_backtesting.py | 4 +- 4 files changed, 394 insertions(+), 378 deletions(-) create mode 100644 freqtrade/optimize/optimize_reports/visualization.py diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py index df42e03bb..4e7375ab4 100644 --- a/freqtrade/optimize/optimize_reports/__init__.py +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -4,7 +4,13 @@ from freqtrade.optimize.optimize_reports.optimize_reports import ( generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, generate_periodic_breakdown_stats, generate_rejected_signals, generate_strategy_comparison, generate_strategy_stats, generate_tag_metrics, generate_trade_signal_candles, - generate_trading_stats, generate_wins_draws_losses, show_backtest_result, show_backtest_results, - show_sorted_pairlist, store_backtest_analysis_results, store_backtest_stats, - text_table_add_metrics, text_table_bt_results, text_table_exit_reason, - text_table_periodic_breakdown, text_table_strategy, text_table_tags) + generate_trading_stats, generate_wins_draws_losses, store_backtest_analysis_results, + store_backtest_stats) +from freqtrade.optimize.optimize_reports.visualization import (show_backtest_result, + show_backtest_results, + show_sorted_pairlist, + text_table_add_metrics, + text_table_bt_results, + text_table_exit_reason, + text_table_periodic_breakdown, + text_table_strategy, text_table_tags) diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index e60047a79..6bf7da329 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -8,7 +8,7 @@ from pandas import DataFrame, concat, to_datetime from tabulate import tabulate from freqtrade.constants import (BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, - UNLIMITED_STAKE_AMOUNT, Config, IntOrInf) + IntOrInf) from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum, calculate_expectancy, calculate_market_change, calculate_max_drawdown, calculate_sharpe, calculate_sortino) @@ -120,24 +120,6 @@ def generate_rejected_signals(preprocessed_df: Dict[str, DataFrame], return rejected_candles_only -def _get_line_floatfmt(stake_currency: str) -> List[str]: - """ - Generate floatformat (goes in line with _generate_result_line()) - """ - return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f', - '.2f', 'd', 's', 's'] - - -def _get_line_header(first_column: str, stake_currency: str, - direction: str = 'Entries') -> List[str]: - """ - Generate header lines (goes in line with _generate_result_line()) - """ - return [first_column, direction, 'Avg Profit %', 'Cum Profit %', - f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', - 'Win Draw Loss Win%'] - - def generate_wins_draws_losses(wins, draws, losses): if wins > 0 and losses == 0: wl_ratio = '100' @@ -652,357 +634,3 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], result['strategy_comparison'] = strategy_results return result - - -### -# Start output section -### - -def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: str) -> str: - """ - Generates and returns a text table for the given backtest data and the results dataframe - :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row - :param stake_currency: stake-currency - used to correctly name headers - :return: pretty printed table with tabulate as string - """ - - headers = _get_line_header('Pair', stake_currency) - floatfmt = _get_line_floatfmt(stake_currency) - output = [[ - t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], - t['profit_total_pct'], t['duration_avg'], - generate_wins_draws_losses(t['wins'], t['draws'], t['losses']) - ] for t in pair_results] - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(output, headers=headers, - floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") - - -def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str: - """ - Generate small table outlining Backtest results - :param sell_reason_stats: Exit reason metrics - :param stake_currency: Stakecurrency used - :return: pretty printed table with tabulate as string - """ - headers = [ - 'Exit Reason', - 'Exits', - 'Win Draws Loss Win%', - 'Avg Profit %', - 'Cum Profit %', - f'Tot Profit {stake_currency}', - 'Tot Profit %', - ] - - output = [[ - t.get('exit_reason', t.get('sell_reason')), t['trades'], - generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), - t['profit_mean_pct'], t['profit_sum_pct'], - round_coin_value(t['profit_total_abs'], stake_currency, False), - t['profit_total_pct'], - ] for t in exit_reason_stats] - return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") - - -def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_currency: str) -> str: - """ - Generates and returns a text table for the given backtest data and the results dataframe - :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row - :param stake_currency: stake-currency - used to correctly name headers - :return: pretty printed table with tabulate as string - """ - if (tag_type == "enter_tag"): - headers = _get_line_header("TAG", stake_currency) - else: - headers = _get_line_header("TAG", stake_currency, 'Exits') - floatfmt = _get_line_floatfmt(stake_currency) - output = [ - [ - t['key'] if t['key'] is not None and len( - t['key']) > 0 else "OTHER", - t['trades'], - t['profit_mean_pct'], - t['profit_sum_pct'], - t['profit_total_abs'], - t['profit_total_pct'], - t['duration_avg'], - generate_wins_draws_losses( - t['wins'], - t['draws'], - t['losses'])] for t in tag_results] - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(output, headers=headers, - floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") - - -def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]], - stake_currency: str, period: str) -> str: - """ - Generate small table with Backtest results by days - :param days_breakdown_stats: Days breakdown metrics - :param stake_currency: Stakecurrency used - :return: pretty printed table with tabulate as string - """ - headers = [ - period.capitalize(), - f'Tot Profit {stake_currency}', - 'Wins', - 'Draws', - 'Losses', - ] - output = [[ - d['date'], round_coin_value(d['profit_abs'], stake_currency, False), - d['wins'], d['draws'], d['loses'], - ] for d in days_breakdown_stats] - return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") - - -def text_table_strategy(strategy_results, stake_currency: str) -> str: - """ - Generate summary table per strategy - :param strategy_results: Dict of containing results for all strategies - :param stake_currency: stake-currency - used to correctly name headers - :return: pretty printed table with tabulate as string - """ - floatfmt = _get_line_floatfmt(stake_currency) - headers = _get_line_header('Strategy', stake_currency) - # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless - # therefore we slip this column in only for strategy summary here. - headers.append('Drawdown') - - # Align drawdown string on the center two space separator. - if 'max_drawdown_account' in strategy_results[0]: - drawdown = [f'{t["max_drawdown_account"] * 100:.2f}' for t in strategy_results] - else: - # Support for prior backtest results - drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results] - - dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results]) - dd_pad_per = max([len(dd) for dd in drawdown]) - drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%' - for t, dd in zip(strategy_results, drawdown)] - - output = [[ - t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], - t['profit_total_pct'], t['duration_avg'], - generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown] - for t, drawdown in zip(strategy_results, drawdown)] - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(output, headers=headers, - floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") - - -def text_table_add_metrics(strat_results: Dict) -> str: - if len(strat_results['trades']) > 0: - best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio']) - worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio']) - - short_metrics = [ - ('', ''), # Empty line to improve readability - ('Long / Short', - f"{strat_results.get('trade_count_long', 'total_trades')} / " - f"{strat_results.get('trade_count_short', 0)}"), - ('Total profit Long %', f"{strat_results['profit_total_long']:.2%}"), - ('Total profit Short %', f"{strat_results['profit_total_short']:.2%}"), - ('Absolute profit Long', round_coin_value(strat_results['profit_total_long_abs'], - strat_results['stake_currency'])), - ('Absolute profit Short', round_coin_value(strat_results['profit_total_short_abs'], - strat_results['stake_currency'])), - ] if strat_results.get('trade_count_short', 0) > 0 else [] - - drawdown_metrics = [] - if 'max_relative_drawdown' in strat_results: - # Compatibility to show old hyperopt results - drawdown_metrics.append( - ('Max % of account underwater', f"{strat_results['max_relative_drawdown']:.2%}") - ) - drawdown_metrics.extend([ - ('Absolute Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}") - if 'max_drawdown_account' in strat_results else ( - 'Drawdown', f"{strat_results['max_drawdown']:.2%}"), - ('Absolute Drawdown', round_coin_value(strat_results['max_drawdown_abs'], - strat_results['stake_currency'])), - ('Drawdown high', round_coin_value(strat_results['max_drawdown_high'], - strat_results['stake_currency'])), - ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'], - strat_results['stake_currency'])), - ('Drawdown Start', strat_results['drawdown_start']), - ('Drawdown End', strat_results['drawdown_end']), - ]) - - entry_adjustment_metrics = [ - ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')), - ('Canceled Entry Orders', strat_results.get('canceled_entry_orders', 'N/A')), - ('Replaced Entry Orders', strat_results.get('replaced_entry_orders', 'N/A')), - ] if strat_results.get('canceled_entry_orders', 0) > 0 else [] - - # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show - # command stores these results and newer version of freqtrade must be able to handle old - # results with missing new fields. - metrics = [ - ('Backtesting from', strat_results['backtest_start']), - ('Backtesting to', strat_results['backtest_end']), - ('Max open trades', strat_results['max_open_trades']), - ('', ''), # Empty line to improve readability - ('Total/Daily Avg Trades', - f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), - - ('Starting balance', round_coin_value(strat_results['starting_balance'], - strat_results['stake_currency'])), - ('Final balance', round_coin_value(strat_results['final_balance'], - strat_results['stake_currency'])), - ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], - strat_results['stake_currency'])), - ('Total profit %', f"{strat_results['profit_total']:.2%}"), - ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), - ('Sortino', f"{strat_results['sortino']:.2f}" if 'sortino' in strat_results else 'N/A'), - ('Sharpe', f"{strat_results['sharpe']:.2f}" if 'sharpe' in strat_results else 'N/A'), - ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'), - ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor' - in strat_results else 'N/A'), - ('Expectancy', f"{strat_results['expectancy']:.2f}" if 'expectancy' - in strat_results else 'N/A'), - ('Trades per day', strat_results['trades_per_day']), - ('Avg. daily profit %', - f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), - ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], - strat_results['stake_currency'])), - ('Total trade volume', round_coin_value(strat_results['total_volume'], - strat_results['stake_currency'])), - *short_metrics, - ('', ''), # Empty line to improve readability - ('Best Pair', f"{strat_results['best_pair']['key']} " - f"{strat_results['best_pair']['profit_sum']:.2%}"), - ('Worst Pair', f"{strat_results['worst_pair']['key']} " - f"{strat_results['worst_pair']['profit_sum']:.2%}"), - ('Best trade', f"{best_trade['pair']} {best_trade['profit_ratio']:.2%}"), - ('Worst trade', f"{worst_trade['pair']} " - f"{worst_trade['profit_ratio']:.2%}"), - - ('Best day', round_coin_value(strat_results['backtest_best_day_abs'], - strat_results['stake_currency'])), - ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], - strat_results['stake_currency'])), - ('Days win/draw/lose', f"{strat_results['winning_days']} / " - f"{strat_results['draw_days']} / {strat_results['losing_days']}"), - ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), - ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), - ('Rejected Entry signals', strat_results.get('rejected_signals', 'N/A')), - ('Entry/Exit Timeouts', - f"{strat_results.get('timedout_entry_orders', 'N/A')} / " - f"{strat_results.get('timedout_exit_orders', 'N/A')}"), - *entry_adjustment_metrics, - ('', ''), # Empty line to improve readability - - ('Min balance', round_coin_value(strat_results['csum_min'], - strat_results['stake_currency'])), - ('Max balance', round_coin_value(strat_results['csum_max'], - strat_results['stake_currency'])), - - *drawdown_metrics, - ('Market change', f"{strat_results['market_change']:.2%}"), - ] - - return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl") - else: - start_balance = round_coin_value(strat_results['starting_balance'], - strat_results['stake_currency']) - stake_amount = round_coin_value( - strat_results['stake_amount'], strat_results['stake_currency'] - ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' - - message = ("No trades made. " - f"Your starting balance was {start_balance}, " - f"and your stake was {stake_amount}." - ) - return message - - -def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str, - backtest_breakdown=[]): - """ - Print results for one strategy - """ - # Print results - print(f"Result for strategy {strategy}") - table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency) - if isinstance(table, str): - print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) - print(table) - - table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) - print(table) - - if (results.get('results_per_enter_tag') is not None - or results.get('results_per_buy_tag') is not None): - # results_per_buy_tag is deprecated and should be removed 2 versions after short golive. - table = text_table_tags( - "enter_tag", - results.get('results_per_enter_tag', results.get('results_per_buy_tag')), - stake_currency=stake_currency) - - if isinstance(table, str) and len(table) > 0: - print(' ENTER TAG STATS '.center(len(table.splitlines()[0]), '=')) - print(table) - - exit_reasons = results.get('exit_reason_summary', results.get('sell_reason_summary')) - table = text_table_exit_reason(exit_reason_stats=exit_reasons, - stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '=')) - print(table) - - for period in backtest_breakdown: - if period in results.get('periodic_breakdown', {}): - days_breakdown_stats = results['periodic_breakdown'][period] - else: - days_breakdown_stats = generate_periodic_breakdown_stats( - trade_list=results['trades'], period=period) - table = text_table_periodic_breakdown(days_breakdown_stats=days_breakdown_stats, - stake_currency=stake_currency, period=period) - if isinstance(table, str) and len(table) > 0: - print(f' {period.upper()} BREAKDOWN '.center(len(table.splitlines()[0]), '=')) - print(table) - - table = text_table_add_metrics(results) - if isinstance(table, str) and len(table) > 0: - print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) - print(table) - - if isinstance(table, str) and len(table) > 0: - print('=' * len(table.splitlines()[0])) - - print() - - -def show_backtest_results(config: Config, backtest_stats: Dict): - stake_currency = config['stake_currency'] - - for strategy, results in backtest_stats['strategy'].items(): - show_backtest_result( - strategy, results, stake_currency, - config.get('backtest_breakdown', [])) - - if len(backtest_stats['strategy']) > 0: - # Print Strategy summary table - - table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency) - print(f"Backtested {results['backtest_start']} -> {results['backtest_end']} |" - f" Max open trades : {results['max_open_trades']}") - print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) - print(table) - print('=' * len(table.splitlines()[0])) - print('\nFor more details, please look at the detail tables above') - - -def show_sorted_pairlist(config: Config, backtest_stats: Dict): - if config.get('backtest_show_pair_list', False): - for strategy, results in backtest_stats['strategy'].items(): - print(f"Pairs for Strategy {strategy}: \n[") - for result in results['results_per_pair']: - if result["key"] != 'TOTAL': - print(f'"{result["key"]}", // {result["profit_mean"]:.2%}') - print("]") diff --git a/freqtrade/optimize/optimize_reports/visualization.py b/freqtrade/optimize/optimize_reports/visualization.py new file mode 100644 index 000000000..c9d4478b3 --- /dev/null +++ b/freqtrade/optimize/optimize_reports/visualization.py @@ -0,0 +1,380 @@ +import logging +from typing import Any, Dict, List + +from tabulate import tabulate + +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config +from freqtrade.misc import decimals_per_coin, round_coin_value +from freqtrade.optimize.optimize_reports.optimize_reports import (generate_periodic_breakdown_stats, + generate_wins_draws_losses) + + +logger = logging.getLogger(__name__) + + +def _get_line_floatfmt(stake_currency: str) -> List[str]: + """ + Generate floatformat (goes in line with _generate_result_line()) + """ + return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f', + '.2f', 'd', 's', 's'] + + +def _get_line_header(first_column: str, stake_currency: str, + direction: str = 'Entries') -> List[str]: + """ + Generate header lines (goes in line with _generate_result_line()) + """ + return [first_column, direction, 'Avg Profit %', 'Cum Profit %', + f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', + 'Win Draw Loss Win%'] + + +def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: str) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row + :param stake_currency: stake-currency - used to correctly name headers + :return: pretty printed table with tabulate as string + """ + + headers = _get_line_header('Pair', stake_currency) + floatfmt = _get_line_floatfmt(stake_currency) + output = [[ + t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], + t['profit_total_pct'], t['duration_avg'], + generate_wins_draws_losses(t['wins'], t['draws'], t['losses']) + ] for t in pair_results] + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(output, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") + + +def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str: + """ + Generate small table outlining Backtest results + :param sell_reason_stats: Exit reason metrics + :param stake_currency: Stakecurrency used + :return: pretty printed table with tabulate as string + """ + headers = [ + 'Exit Reason', + 'Exits', + 'Win Draws Loss Win%', + 'Avg Profit %', + 'Cum Profit %', + f'Tot Profit {stake_currency}', + 'Tot Profit %', + ] + + output = [[ + t.get('exit_reason', t.get('sell_reason')), t['trades'], + generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), + t['profit_mean_pct'], t['profit_sum_pct'], + round_coin_value(t['profit_total_abs'], stake_currency, False), + t['profit_total_pct'], + ] for t in exit_reason_stats] + return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") + + +def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_currency: str) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :param pair_results: List of Dictionaries - one entry per pair + final TOTAL row + :param stake_currency: stake-currency - used to correctly name headers + :return: pretty printed table with tabulate as string + """ + if (tag_type == "enter_tag"): + headers = _get_line_header("TAG", stake_currency) + else: + headers = _get_line_header("TAG", stake_currency, 'Exits') + floatfmt = _get_line_floatfmt(stake_currency) + output = [ + [ + t['key'] if t['key'] is not None and len( + t['key']) > 0 else "OTHER", + t['trades'], + t['profit_mean_pct'], + t['profit_sum_pct'], + t['profit_total_abs'], + t['profit_total_pct'], + t['duration_avg'], + generate_wins_draws_losses( + t['wins'], + t['draws'], + t['losses'])] for t in tag_results] + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(output, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") + + +def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]], + stake_currency: str, period: str) -> str: + """ + Generate small table with Backtest results by days + :param days_breakdown_stats: Days breakdown metrics + :param stake_currency: Stakecurrency used + :return: pretty printed table with tabulate as string + """ + headers = [ + period.capitalize(), + f'Tot Profit {stake_currency}', + 'Wins', + 'Draws', + 'Losses', + ] + output = [[ + d['date'], round_coin_value(d['profit_abs'], stake_currency, False), + d['wins'], d['draws'], d['loses'], + ] for d in days_breakdown_stats] + return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") + + +def text_table_strategy(strategy_results, stake_currency: str) -> str: + """ + Generate summary table per strategy + :param strategy_results: Dict of containing results for all strategies + :param stake_currency: stake-currency - used to correctly name headers + :return: pretty printed table with tabulate as string + """ + floatfmt = _get_line_floatfmt(stake_currency) + headers = _get_line_header('Strategy', stake_currency) + # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless + # therefore we slip this column in only for strategy summary here. + headers.append('Drawdown') + + # Align drawdown string on the center two space separator. + if 'max_drawdown_account' in strategy_results[0]: + drawdown = [f'{t["max_drawdown_account"] * 100:.2f}' for t in strategy_results] + else: + # Support for prior backtest results + drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results] + + dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results]) + dd_pad_per = max([len(dd) for dd in drawdown]) + drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%' + for t, dd in zip(strategy_results, drawdown)] + + output = [[ + t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], + t['profit_total_pct'], t['duration_avg'], + generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown] + for t, drawdown in zip(strategy_results, drawdown)] + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(output, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") + + +def text_table_add_metrics(strat_results: Dict) -> str: + if len(strat_results['trades']) > 0: + best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio']) + worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio']) + + short_metrics = [ + ('', ''), # Empty line to improve readability + ('Long / Short', + f"{strat_results.get('trade_count_long', 'total_trades')} / " + f"{strat_results.get('trade_count_short', 0)}"), + ('Total profit Long %', f"{strat_results['profit_total_long']:.2%}"), + ('Total profit Short %', f"{strat_results['profit_total_short']:.2%}"), + ('Absolute profit Long', round_coin_value(strat_results['profit_total_long_abs'], + strat_results['stake_currency'])), + ('Absolute profit Short', round_coin_value(strat_results['profit_total_short_abs'], + strat_results['stake_currency'])), + ] if strat_results.get('trade_count_short', 0) > 0 else [] + + drawdown_metrics = [] + if 'max_relative_drawdown' in strat_results: + # Compatibility to show old hyperopt results + drawdown_metrics.append( + ('Max % of account underwater', f"{strat_results['max_relative_drawdown']:.2%}") + ) + drawdown_metrics.extend([ + ('Absolute Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}") + if 'max_drawdown_account' in strat_results else ( + 'Drawdown', f"{strat_results['max_drawdown']:.2%}"), + ('Absolute Drawdown', round_coin_value(strat_results['max_drawdown_abs'], + strat_results['stake_currency'])), + ('Drawdown high', round_coin_value(strat_results['max_drawdown_high'], + strat_results['stake_currency'])), + ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'], + strat_results['stake_currency'])), + ('Drawdown Start', strat_results['drawdown_start']), + ('Drawdown End', strat_results['drawdown_end']), + ]) + + entry_adjustment_metrics = [ + ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')), + ('Canceled Entry Orders', strat_results.get('canceled_entry_orders', 'N/A')), + ('Replaced Entry Orders', strat_results.get('replaced_entry_orders', 'N/A')), + ] if strat_results.get('canceled_entry_orders', 0) > 0 else [] + + # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show + # command stores these results and newer version of freqtrade must be able to handle old + # results with missing new fields. + metrics = [ + ('Backtesting from', strat_results['backtest_start']), + ('Backtesting to', strat_results['backtest_end']), + ('Max open trades', strat_results['max_open_trades']), + ('', ''), # Empty line to improve readability + ('Total/Daily Avg Trades', + f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), + + ('Starting balance', round_coin_value(strat_results['starting_balance'], + strat_results['stake_currency'])), + ('Final balance', round_coin_value(strat_results['final_balance'], + strat_results['stake_currency'])), + ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], + strat_results['stake_currency'])), + ('Total profit %', f"{strat_results['profit_total']:.2%}"), + ('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'), + ('Sortino', f"{strat_results['sortino']:.2f}" if 'sortino' in strat_results else 'N/A'), + ('Sharpe', f"{strat_results['sharpe']:.2f}" if 'sharpe' in strat_results else 'N/A'), + ('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'), + ('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor' + in strat_results else 'N/A'), + ('Expectancy', f"{strat_results['expectancy']:.2f}" if 'expectancy' + in strat_results else 'N/A'), + ('Trades per day', strat_results['trades_per_day']), + ('Avg. daily profit %', + f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"), + ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], + strat_results['stake_currency'])), + ('Total trade volume', round_coin_value(strat_results['total_volume'], + strat_results['stake_currency'])), + *short_metrics, + ('', ''), # Empty line to improve readability + ('Best Pair', f"{strat_results['best_pair']['key']} " + f"{strat_results['best_pair']['profit_sum']:.2%}"), + ('Worst Pair', f"{strat_results['worst_pair']['key']} " + f"{strat_results['worst_pair']['profit_sum']:.2%}"), + ('Best trade', f"{best_trade['pair']} {best_trade['profit_ratio']:.2%}"), + ('Worst trade', f"{worst_trade['pair']} " + f"{worst_trade['profit_ratio']:.2%}"), + + ('Best day', round_coin_value(strat_results['backtest_best_day_abs'], + strat_results['stake_currency'])), + ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], + strat_results['stake_currency'])), + ('Days win/draw/lose', f"{strat_results['winning_days']} / " + f"{strat_results['draw_days']} / {strat_results['losing_days']}"), + ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), + ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), + ('Rejected Entry signals', strat_results.get('rejected_signals', 'N/A')), + ('Entry/Exit Timeouts', + f"{strat_results.get('timedout_entry_orders', 'N/A')} / " + f"{strat_results.get('timedout_exit_orders', 'N/A')}"), + *entry_adjustment_metrics, + ('', ''), # Empty line to improve readability + + ('Min balance', round_coin_value(strat_results['csum_min'], + strat_results['stake_currency'])), + ('Max balance', round_coin_value(strat_results['csum_max'], + strat_results['stake_currency'])), + + *drawdown_metrics, + ('Market change', f"{strat_results['market_change']:.2%}"), + ] + + return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl") + else: + start_balance = round_coin_value(strat_results['starting_balance'], + strat_results['stake_currency']) + stake_amount = round_coin_value( + strat_results['stake_amount'], strat_results['stake_currency'] + ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' + + message = ("No trades made. " + f"Your starting balance was {start_balance}, " + f"and your stake was {stake_amount}." + ) + return message + + +def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str, + backtest_breakdown=[]): + """ + Print results for one strategy + """ + # Print results + print(f"Result for strategy {strategy}") + table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + + if (results.get('results_per_enter_tag') is not None + or results.get('results_per_buy_tag') is not None): + # results_per_buy_tag is deprecated and should be removed 2 versions after short golive. + table = text_table_tags( + "enter_tag", + results.get('results_per_enter_tag', results.get('results_per_buy_tag')), + stake_currency=stake_currency) + + if isinstance(table, str) and len(table) > 0: + print(' ENTER TAG STATS '.center(len(table.splitlines()[0]), '=')) + print(table) + + exit_reasons = results.get('exit_reason_summary', results.get('sell_reason_summary')) + table = text_table_exit_reason(exit_reason_stats=exit_reasons, + stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) + + for period in backtest_breakdown: + if period in results.get('periodic_breakdown', {}): + days_breakdown_stats = results['periodic_breakdown'][period] + else: + days_breakdown_stats = generate_periodic_breakdown_stats( + trade_list=results['trades'], period=period) + table = text_table_periodic_breakdown(days_breakdown_stats=days_breakdown_stats, + stake_currency=stake_currency, period=period) + if isinstance(table, str) and len(table) > 0: + print(f' {period.upper()} BREAKDOWN '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = text_table_add_metrics(results) + if isinstance(table, str) and len(table) > 0: + print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) + print(table) + + if isinstance(table, str) and len(table) > 0: + print('=' * len(table.splitlines()[0])) + + print() + + +def show_backtest_results(config: Config, backtest_stats: Dict): + stake_currency = config['stake_currency'] + + for strategy, results in backtest_stats['strategy'].items(): + show_backtest_result( + strategy, results, stake_currency, + config.get('backtest_breakdown', [])) + + if len(backtest_stats['strategy']) > 0: + # Print Strategy summary table + + table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency) + print(f"Backtested {results['backtest_start']} -> {results['backtest_end']} |" + f" Max open trades : {results['max_open_trades']}") + print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) + print(table) + print('=' * len(table.splitlines()[0])) + print('\nFor more details, please look at the detail tables above') + + +def show_sorted_pairlist(config: Config, backtest_stats: Dict): + if config.get('backtest_show_pair_list', False): + for strategy, results in backtest_stats['strategy'].items(): + print(f"Pairs for Strategy {strategy}: \n[") + for result in results['results_per_pair']: + if result["key"] != 'TOTAL': + print(f'"{result["key"]}", // {result["profit_mean"]:.2%}') + print("]") diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 15be6bb28..5dc3121e7 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1437,9 +1437,11 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): strattable_mock = MagicMock() strat_summary = MagicMock() - mocker.patch.multiple('freqtrade.optimize.optimize_reports.optimize_reports', + mocker.patch.multiple('freqtrade.optimize.optimize_reports.visualization', text_table_bt_results=text_table_mock, text_table_strategy=strattable_mock, + ) + mocker.patch.multiple('freqtrade.optimize.optimize_reports.optimize_reports', generate_pair_metrics=MagicMock(), generate_exit_reason_stats=sell_reason_mock, generate_strategy_comparison=strat_summary, From 65e8359908008f5c84827e0bdaaf92113c8534b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 17:11:13 +0200 Subject: [PATCH 27/57] Improve naming of new file --- freqtrade/optimize/optimize_reports/__init__.py | 16 ++++++++-------- .../{visualization.py => bt_output.py} | 0 tests/optimize/test_backtesting.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename freqtrade/optimize/optimize_reports/{visualization.py => bt_output.py} (100%) diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py index 4e7375ab4..767fbcba3 100644 --- a/freqtrade/optimize/optimize_reports/__init__.py +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -1,4 +1,12 @@ # flake8: noqa: F401 +from freqtrade.optimize.optimize_reports.bt_output import (show_backtest_result, + show_backtest_results, + show_sorted_pairlist, + text_table_add_metrics, + text_table_bt_results, + text_table_exit_reason, + text_table_periodic_breakdown, + text_table_strategy, text_table_tags) from freqtrade.optimize.optimize_reports.optimize_reports import ( generate_all_periodic_breakdown_stats, generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, @@ -6,11 +14,3 @@ from freqtrade.optimize.optimize_reports.optimize_reports import ( generate_strategy_stats, generate_tag_metrics, generate_trade_signal_candles, generate_trading_stats, generate_wins_draws_losses, store_backtest_analysis_results, store_backtest_stats) -from freqtrade.optimize.optimize_reports.visualization import (show_backtest_result, - show_backtest_results, - show_sorted_pairlist, - text_table_add_metrics, - text_table_bt_results, - text_table_exit_reason, - text_table_periodic_breakdown, - text_table_strategy, text_table_tags) diff --git a/freqtrade/optimize/optimize_reports/visualization.py b/freqtrade/optimize/optimize_reports/bt_output.py similarity index 100% rename from freqtrade/optimize/optimize_reports/visualization.py rename to freqtrade/optimize/optimize_reports/bt_output.py diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5dc3121e7..a333cda9d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1437,7 +1437,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): strattable_mock = MagicMock() strat_summary = MagicMock() - mocker.patch.multiple('freqtrade.optimize.optimize_reports.visualization', + mocker.patch.multiple('freqtrade.optimize.optimize_reports.bt_output', text_table_bt_results=text_table_mock, text_table_strategy=strattable_mock, ) From 72504e62ad833daa1cb2680cc74d3fba53f6e55a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 17:42:58 +0200 Subject: [PATCH 28/57] Extract btstorage methods --- .../optimize/optimize_reports/__init__.py | 5 +- .../optimize/optimize_reports/bt_storage.py | 71 +++++++++++++++++++ .../optimize_reports/optimize_reports.py | 66 +---------------- tests/optimize/test_optimize_reports.py | 4 +- 4 files changed, 78 insertions(+), 68 deletions(-) create mode 100644 freqtrade/optimize/optimize_reports/bt_storage.py diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py index 767fbcba3..c8865d84d 100644 --- a/freqtrade/optimize/optimize_reports/__init__.py +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -7,10 +7,11 @@ from freqtrade.optimize.optimize_reports.bt_output import (show_backtest_result, text_table_exit_reason, text_table_periodic_breakdown, text_table_strategy, text_table_tags) +from freqtrade.optimize.optimize_reports.bt_storage import (store_backtest_analysis_results, + store_backtest_stats) from freqtrade.optimize.optimize_reports.optimize_reports import ( generate_all_periodic_breakdown_stats, generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, generate_periodic_breakdown_stats, generate_rejected_signals, generate_strategy_comparison, generate_strategy_stats, generate_tag_metrics, generate_trade_signal_candles, - generate_trading_stats, generate_wins_draws_losses, store_backtest_analysis_results, - store_backtest_stats) + generate_trading_stats, generate_wins_draws_losses) diff --git a/freqtrade/optimize/optimize_reports/bt_storage.py b/freqtrade/optimize/optimize_reports/bt_storage.py new file mode 100644 index 000000000..af97753e3 --- /dev/null +++ b/freqtrade/optimize/optimize_reports/bt_storage.py @@ -0,0 +1,71 @@ +import logging +from pathlib import Path +from typing import Dict + +from pandas import DataFrame + +from freqtrade.constants import LAST_BT_RESULT_FN +from freqtrade.misc import file_dump_joblib, file_dump_json +from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename + + +logger = logging.getLogger(__name__) + + +def store_backtest_stats( + recordfilename: Path, stats: Dict[str, DataFrame], dtappendix: str) -> None: + """ + Stores backtest results + :param recordfilename: Path object, which can either be a filename or a directory. + Filenames will be appended with a timestamp right before the suffix + while for directories, /backtest-result-.json will be used as filename + :param stats: Dataframe containing the backtesting statistics + :param dtappendix: Datetime to use for the filename + """ + if recordfilename.is_dir(): + filename = (recordfilename / f'backtest-result-{dtappendix}.json') + else: + filename = Path.joinpath( + recordfilename.parent, f'{recordfilename.stem}-{dtappendix}' + ).with_suffix(recordfilename.suffix) + + # Store metadata separately. + file_dump_json(get_backtest_metadata_filename(filename), stats['metadata']) + del stats['metadata'] + + file_dump_json(filename, stats) + + latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) + + +def _store_backtest_analysis_data( + recordfilename: Path, data: Dict[str, Dict], + dtappendix: str, name: str) -> Path: + """ + Stores backtest trade candles for analysis + :param recordfilename: Path object, which can either be a filename or a directory. + Filenames will be appended with a timestamp right before the suffix + while for directories, /backtest-result-_.pkl will be used + as filename + :param candles: Dict containing the backtesting data for analysis + :param dtappendix: Datetime to use for the filename + :param name: Name to use for the file, e.g. signals, rejected + """ + if recordfilename.is_dir(): + filename = (recordfilename / f'backtest-result-{dtappendix}_{name}.pkl') + else: + filename = Path.joinpath( + recordfilename.parent, f'{recordfilename.stem}-{dtappendix}_{name}.pkl' + ) + + file_dump_joblib(filename, data) + + return filename + + +def store_backtest_analysis_results( + recordfilename: Path, candles: Dict[str, Dict], trades: Dict[str, Dict], + dtappendix: str) -> None: + _store_backtest_analysis_data(recordfilename, candles, dtappendix, "signals") + _store_backtest_analysis_data(recordfilename, trades, dtappendix, "rejected") diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index 6bf7da329..c217fe5f9 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -1,83 +1,21 @@ import logging from copy import deepcopy from datetime import datetime, timedelta, timezone -from pathlib import Path from typing import Any, Dict, List, Union from pandas import DataFrame, concat, to_datetime from tabulate import tabulate -from freqtrade.constants import (BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, - IntOrInf) +from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum, calculate_expectancy, calculate_market_change, calculate_max_drawdown, calculate_sharpe, calculate_sortino) -from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value -from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename +from freqtrade.misc import decimals_per_coin, round_coin_value logger = logging.getLogger(__name__) -def store_backtest_stats( - recordfilename: Path, stats: Dict[str, DataFrame], dtappendix: str) -> None: - """ - Stores backtest results - :param recordfilename: Path object, which can either be a filename or a directory. - Filenames will be appended with a timestamp right before the suffix - while for directories, /backtest-result-.json will be used as filename - :param stats: Dataframe containing the backtesting statistics - :param dtappendix: Datetime to use for the filename - """ - if recordfilename.is_dir(): - filename = (recordfilename / f'backtest-result-{dtappendix}.json') - else: - filename = Path.joinpath( - recordfilename.parent, f'{recordfilename.stem}-{dtappendix}' - ).with_suffix(recordfilename.suffix) - - # Store metadata separately. - file_dump_json(get_backtest_metadata_filename(filename), stats['metadata']) - del stats['metadata'] - - file_dump_json(filename, stats) - - latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) - file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) - - -def _store_backtest_analysis_data( - recordfilename: Path, data: Dict[str, Dict], - dtappendix: str, name: str) -> Path: - """ - Stores backtest trade candles for analysis - :param recordfilename: Path object, which can either be a filename or a directory. - Filenames will be appended with a timestamp right before the suffix - while for directories, /backtest-result-_.pkl will be used - as filename - :param candles: Dict containing the backtesting data for analysis - :param dtappendix: Datetime to use for the filename - :param name: Name to use for the file, e.g. signals, rejected - """ - if recordfilename.is_dir(): - filename = (recordfilename / f'backtest-result-{dtappendix}_{name}.pkl') - else: - filename = Path.joinpath( - recordfilename.parent, f'{recordfilename.stem}-{dtappendix}_{name}.pkl' - ) - - file_dump_joblib(filename, data) - - return filename - - -def store_backtest_analysis_results( - recordfilename: Path, candles: Dict[str, Dict], trades: Dict[str, Dict], - dtappendix: str) -> None: - _store_backtest_analysis_data(recordfilename, candles, dtappendix, "signals") - _store_backtest_analysis_data(recordfilename, trades, dtappendix, "rejected") - - def generate_trade_signal_candles(preprocessed_df: Dict[str, DataFrame], bt_results: Dict[str, Any]) -> DataFrame: signal_candles_only = {} diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 1ea2a7380..7b85e7978 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -210,7 +210,7 @@ def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): def test_store_backtest_stats(testdatadir, mocker): - dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.optimize_reports.file_dump_json') + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.bt_storage.file_dump_json') store_backtest_stats(testdatadir, {'metadata': {}}, '2022_01_01_15_05_13') @@ -230,7 +230,7 @@ def test_store_backtest_stats(testdatadir, mocker): def test_store_backtest_candles(testdatadir, mocker): dump_mock = mocker.patch( - 'freqtrade.optimize.optimize_reports.optimize_reports.file_dump_joblib') + 'freqtrade.optimize.optimize_reports.bt_storage.file_dump_joblib') candle_dict = {'DefStrat': {'UNITTEST/BTC': pd.DataFrame()}} From 1717f867020dd7a451ca7d0ad702e5ea4ae0559d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 25 Jun 2023 17:45:01 +0200 Subject: [PATCH 29/57] Extract edge output to proper module --- .../optimize/optimize_reports/__init__.py | 11 ++++---- .../optimize/optimize_reports/bt_output.py | 25 ++++++++++++++++++ .../optimize_reports/optimize_reports.py | 26 ------------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/freqtrade/optimize/optimize_reports/__init__.py b/freqtrade/optimize/optimize_reports/__init__.py index c8865d84d..68e222d00 100644 --- a/freqtrade/optimize/optimize_reports/__init__.py +++ b/freqtrade/optimize/optimize_reports/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 -from freqtrade.optimize.optimize_reports.bt_output import (show_backtest_result, +from freqtrade.optimize.optimize_reports.bt_output import (generate_edge_table, + show_backtest_result, show_backtest_results, show_sorted_pairlist, text_table_add_metrics, @@ -11,7 +12,7 @@ from freqtrade.optimize.optimize_reports.bt_storage import (store_backtest_analy store_backtest_stats) from freqtrade.optimize.optimize_reports.optimize_reports import ( generate_all_periodic_breakdown_stats, generate_backtest_stats, generate_daily_stats, - generate_edge_table, generate_exit_reason_stats, generate_pair_metrics, - generate_periodic_breakdown_stats, generate_rejected_signals, generate_strategy_comparison, - generate_strategy_stats, generate_tag_metrics, generate_trade_signal_candles, - generate_trading_stats, generate_wins_draws_losses) + generate_exit_reason_stats, generate_pair_metrics, generate_periodic_breakdown_stats, + generate_rejected_signals, generate_strategy_comparison, generate_strategy_stats, + generate_tag_metrics, generate_trade_signal_candles, generate_trading_stats, + generate_wins_draws_losses) diff --git a/freqtrade/optimize/optimize_reports/bt_output.py b/freqtrade/optimize/optimize_reports/bt_output.py index c9d4478b3..1fd1f7a34 100644 --- a/freqtrade/optimize/optimize_reports/bt_output.py +++ b/freqtrade/optimize/optimize_reports/bt_output.py @@ -378,3 +378,28 @@ def show_sorted_pairlist(config: Config, backtest_stats: Dict): if result["key"] != 'TOTAL': print(f'"{result["key"]}", // {result["profit_mean"]:.2%}') print("]") + + +def generate_edge_table(results: dict) -> str: + floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', 'd', 'd') + tabular_data = [] + headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio', + 'Required Risk Reward', 'Expectancy', 'Total Number of Trades', + 'Average Duration (min)'] + + for result in results.items(): + if result[1].nb_trades > 0: + tabular_data.append([ + result[0], + result[1].stoploss, + result[1].winrate, + result[1].risk_reward_ratio, + result[1].required_risk_reward, + result[1].expectancy, + result[1].nb_trades, + round(result[1].avg_trade_duration) + ]) + + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(tabular_data, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") diff --git a/freqtrade/optimize/optimize_reports/optimize_reports.py b/freqtrade/optimize/optimize_reports/optimize_reports.py index c217fe5f9..015f163e3 100644 --- a/freqtrade/optimize/optimize_reports/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports/optimize_reports.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Union from pandas import DataFrame, concat, to_datetime -from tabulate import tabulate from freqtrade.constants import BACKTEST_BREAKDOWNS, DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum, @@ -215,31 +214,6 @@ def generate_strategy_comparison(bt_stats: Dict) -> List[Dict]: return tabular_data -def generate_edge_table(results: dict) -> str: - floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', 'd', 'd') - tabular_data = [] - headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio', - 'Required Risk Reward', 'Expectancy', 'Total Number of Trades', - 'Average Duration (min)'] - - for result in results.items(): - if result[1].nb_trades > 0: - tabular_data.append([ - result[0], - result[1].stoploss, - result[1].winrate, - result[1].risk_reward_ratio, - result[1].required_risk_reward, - result[1].expectancy, - result[1].nb_trades, - round(result[1].avg_trade_duration) - ]) - - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") - - def _get_resample_from_period(period: str) -> str: if period == 'day': return '1d' From fec4cb3cf9ff49db9ec76831977575b0dd5785b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 03:57:07 +0000 Subject: [PATCH 30/57] Bump nbconvert from 7.5.0 to 7.6.0 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.5.0 to 7.6.0. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md) - [Commits](https://github.com/jupyter/nbconvert/compare/v7.5.0...v7.6.0) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b1633fd49..842a19612 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ isort==5.12.0 time-machine==2.10.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.5.0 +nbconvert==7.6.0 # mypy types types-cachetools==5.3.0.5 From 2d2699b0ada88189021da1139e30966a797822db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 03:57:21 +0000 Subject: [PATCH 31/57] Bump mkdocs-material from 9.1.16 to 9.1.17 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.1.16 to 9.1.17. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.1.16...9.1.17) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 5c936a868..d6ee2fd14 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.3 -mkdocs-material==9.1.16 +mkdocs-material==9.1.17 mdx_truly_sane_lists==1.3 pymdown-extensions==10.0.1 jinja2==3.1.2 From ae42d57a2637235f4d1f5d104107466b47307689 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 03:57:31 +0000 Subject: [PATCH 32/57] Bump pytest from 7.3.2 to 7.4.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.2 to 7.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.2...7.4.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b1633fd49..311199fd2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.0.272 mypy==1.3.0 pre-commit==3.3.3 -pytest==7.3.2 +pytest==7.4.0 pytest-asyncio==0.21.0 pytest-cov==4.1.0 pytest-mock==3.11.1 From 6274197f85335fbf081407cefee28180c0c49c2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 03:58:21 +0000 Subject: [PATCH 33/57] Bump sqlalchemy from 2.0.16 to 2.0.17 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.16 to 2.0.17. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a2ae6318..7b95daf33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==3.1.44 cryptography==41.0.1; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 -SQLAlchemy==2.0.16 +SQLAlchemy==2.0.17 python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 From 8c2098c262ed962f9eebcacef7e1782ce1f85aff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 03:58:27 +0000 Subject: [PATCH 34/57] Bump fastapi from 0.97.0 to 0.98.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.97.0 to 0.98.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.97.0...0.98.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a2ae6318..0c7d89e17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ orjson==3.9.1 sdnotify==0.3.2 # API Server -fastapi==0.97.0 +fastapi==0.98.0 pydantic==1.10.9 uvicorn==0.22.0 pyjwt==2.7.0 From 6b201d525ea8c4b2ddfd1edae1fb991bdf33e270 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 26 Jun 2023 14:42:59 +0200 Subject: [PATCH 35/57] make sure default PCA behavior reduces parameter space size --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9fe8bd194..36c94130c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -515,7 +515,7 @@ class IFreqaiModel(ABC): ] if ft_params.get("principal_component_analysis", False): - pipe_steps.append(('pca', ds.PCA())) + pipe_steps.append(('pca', ds.PCA(n_components=0.999))) pipe_steps.append(('post-pca-scaler', SKLearnWrapper(MinMaxScaler(feature_range=(-1, 1))))) From be07ea5d4f92baf0530430470b93dad6683849c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:17:23 +0000 Subject: [PATCH 36/57] Bump mypy from 1.3.0 to 1.4.1 Bumps [mypy](https://github.com/python/mypy) from 1.3.0 to 1.4.1. - [Commits](https://github.com/python/mypy/compare/v1.3.0...v1.4.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6c8cf8fcc..ca9b675ec 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 ruff==0.0.272 -mypy==1.3.0 +mypy==1.4.1 pre-commit==3.3.3 pytest==7.4.0 pytest-asyncio==0.21.0 From b12dbd2bea6e309c4a96198773d28278638e2a65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:17:59 +0000 Subject: [PATCH 37/57] Bump ruff from 0.0.272 to 0.0.275 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.272 to 0.0.275. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.0.272...v0.0.275) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6c8cf8fcc..5da4a404f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.272 +ruff==0.0.275 mypy==1.3.0 pre-commit==3.3.3 pytest==7.4.0 From 4b06b4772d7b56d51884270b7b55c90ed311ff1a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 27 Jun 2023 11:53:58 +0000 Subject: [PATCH 38/57] sqlalchemy - pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 476d63847..56c8a6010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.31.0.1 - types-tabulate==0.9.0.2 - types-python-dateutil==2.8.19.13 - - SQLAlchemy==2.0.16 + - SQLAlchemy==2.0.17 # stages: [push] - repo: https://github.com/pycqa/isort From accc1b509be9543f212d840d0d3108a2153cb374 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Jun 2023 12:16:10 +0000 Subject: [PATCH 39/57] Simplify class setups without inheritance --- freqtrade/mixins/logging_mixin.py | 2 +- freqtrade/optimize/hyperopt_tools.py | 4 ++-- freqtrade/persistence/key_value_store.py | 2 +- freqtrade/persistence/pairlock_middleware.py | 2 +- freqtrade/persistence/trade_model.py | 2 +- freqtrade/plugins/protectionmanager.py | 2 +- freqtrade/rpc/api_server/webserver_bgwork.py | 2 +- freqtrade/strategy/interface.py | 2 +- scripts/rest_client.py | 2 +- tests/exchange/test_ccxt_compat.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index 06935d5f6..31b49ba55 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -3,7 +3,7 @@ from typing import Callable from cachetools import TTLCache, cached -class LoggingMixin(): +class LoggingMixin: """ Logging Mixin Shows similar messages only once every `refresh_period`. diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 1e7befdf6..bc5b85309 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -35,7 +35,7 @@ def hyperopt_serializer(x): return str(x) -class HyperoptStateContainer(): +class HyperoptStateContainer: """ Singleton class to track state of hyperopt""" state: HyperoptState = HyperoptState.OPTIMIZE @@ -44,7 +44,7 @@ class HyperoptStateContainer(): cls.state = value -class HyperoptTools(): +class HyperoptTools: @staticmethod def get_strategy_filename(config: Config, strategy_name: str) -> Optional[Path]: diff --git a/freqtrade/persistence/key_value_store.py b/freqtrade/persistence/key_value_store.py index 110a23d6c..6da7265d6 100644 --- a/freqtrade/persistence/key_value_store.py +++ b/freqtrade/persistence/key_value_store.py @@ -42,7 +42,7 @@ class _KeyValueStoreModel(ModelBase): int_value: Mapped[Optional[int]] -class KeyValueStore(): +class KeyValueStore: """ Generic bot-wide, persistent key-value store Can be used to store generic values, e.g. very first bot startup time. diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 29169a50d..dd6bacf3a 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -11,7 +11,7 @@ from freqtrade.persistence.models import PairLock logger = logging.getLogger(__name__) -class PairLocks(): +class PairLocks: """ Pairlocks middleware class Abstracts the database layer away so it becomes optional - which will be necessary to support diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 4d3564df6..35a44e3fc 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -283,7 +283,7 @@ class Order(ModelBase): return Order.session.scalars(select(Order).filter(Order.order_id == order_id)).first() -class LocalTrade(): +class LocalTrade: """ Trade database model. Used in backtesting - must be aligned to Trade model! diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 54432e677..6e55ade11 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -15,7 +15,7 @@ from freqtrade.resolvers import ProtectionResolver logger = logging.getLogger(__name__) -class ProtectionManager(): +class ProtectionManager: def __init__(self, config: Config, protections: List) -> None: self._config = config diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 3846fe138..13f45227e 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -14,7 +14,7 @@ class JobsContainer(TypedDict): error: Optional[str] -class ApiBG(): +class ApiBG: # Backtesting type: Backtesting bt: Dict[str, Any] = { 'bt': None, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d0655b504..1e9ebe1ae 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -168,7 +168,7 @@ class IStrategy(ABC, HyperStrategyMixin): download_all_data_for_training(self.dp, self.config) else: # Gracious failures if freqAI is disabled but "start" is called. - class DummyClass(): + class DummyClass: def start(self, *args, **kwargs): raise OperationalException( 'freqAI is not enabled. ' diff --git a/scripts/rest_client.py b/scripts/rest_client.py index f9c9858ed..2b4690287 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -29,7 +29,7 @@ logging.basicConfig( logger = logging.getLogger("ft_rest_client") -class FtRestClient(): +class FtRestClient: def __init__(self, serverurl, username=None, password=None): diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 404b51d10..b1a6f1c65 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -342,7 +342,7 @@ def exchange_futures(request, exchange_conf, class_mocker): @pytest.mark.longrun -class TestCCXTExchange(): +class TestCCXTExchange: def test_load_markets(self, exchange: EXCHANGE_FIXTURE_TYPE): exch, exchangename = exchange From 29725440c851bfaa3abfc561b6ec17c067abe192 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 29 Jun 2023 12:08:52 +0000 Subject: [PATCH 40/57] Simplify RPCMessageType schema definition --- freqtrade/constants.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ffcd87744..de1e7aa51 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -112,6 +112,8 @@ MINIMAL_CONFIG = { } } +__MESSAGE_TYPE_DICT: Dict[str, Dict[str, str]] = {x: {'type': 'object'} for x in RPCMessageType} + # Required json-schema for user specified config CONF_SCHEMA = { 'type': 'object', @@ -354,7 +356,8 @@ CONF_SCHEMA = { 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'}, 'retries': {'type': 'integer', 'minimum': 0}, 'retry_delay': {'type': 'number', 'minimum': 0}, - **dict([(x, {'type': 'object'}) for x in RPCMessageType]), + **__MESSAGE_TYPE_DICT, + # **{x: {'type': 'object'} for x in RPCMessageType}, # Below -> Deprecated 'webhookentry': {'type': 'object'}, 'webhookentrycancel': {'type': 'object'}, From b204da3317206586d1ac79191b5dec2b72e20254 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jul 2023 05:43:10 +0000 Subject: [PATCH 41/57] Bump scipy from 1.10.1 to 1.11.1 Bumps [scipy](https://github.com/scipy/scipy) from 1.10.1 to 1.11.1. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.10.1...v1.11.1) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 163fee75f..920da615c 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.10.1 +scipy==1.11.1 scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.12.2 From 0310a26b807458e39b09a14e55e0627031ed34e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Jul 2023 16:44:47 +0000 Subject: [PATCH 42/57] Fix documentation typo --- docs/advanced-hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index ff0521f4f..eb8bf3f84 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -136,7 +136,7 @@ class MyAwesomeStrategy(IStrategy): ### Dynamic parameters -Parameters can also be defined dynamically, but must be available to the instance once the * [`bot_start()` callback](strategy-callbacks.md#bot-start) has been called. +Parameters can also be defined dynamically, but must be available to the instance once the [`bot_start()` callback](strategy-callbacks.md#bot-start) has been called. ``` python From 9d3dda4e12f226cd10959787f901d0efa0d6b794 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 03:29:40 +0000 Subject: [PATCH 43/57] Bump stable-baselines3 from 2.0.0a13 to 2.0.0 Bumps [stable-baselines3](https://github.com/DLR-RM/stable-baselines3) from 2.0.0a13 to 2.0.0. - [Release notes](https://github.com/DLR-RM/stable-baselines3/releases) - [Commits](https://github.com/DLR-RM/stable-baselines3/commits/v2.0.0) --- updated-dependencies: - dependency-name: stable-baselines3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-freqai-rl.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 2672f9c38..74c6d4ebe 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a13 +stable_baselines3==2.0.0 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From 1880f9ffa1b0481deec302c7529f779a61ff8057 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 03:29:48 +0000 Subject: [PATCH 44/57] Bump ast-comments from 1.0.1 to 1.1.0 Bumps [ast-comments](https://github.com/t3rn0/ast-comments) from 1.0.1 to 1.1.0. - [Commits](https://github.com/t3rn0/ast-comments/compare/1.0.1...1.1.0) --- updated-dependencies: - dependency-name: ast-comments dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7c398e3c5..d29b0fdaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,5 +60,5 @@ schedule==1.2.0 websockets==11.0.3 janus==1.0.0 -ast-comments==1.0.1 +ast-comments==1.1.0 packaging==23.1 From ba6cba31be5bfeab096a6283a67641c2c4048265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 03:29:53 +0000 Subject: [PATCH 45/57] Bump fastapi from 0.98.0 to 0.99.1 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.98.0 to 0.99.1. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.98.0...0.99.1) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7c398e3c5..1e8cfbd7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ orjson==3.9.1 sdnotify==0.3.2 # API Server -fastapi==0.98.0 +fastapi==0.99.1 pydantic==1.10.9 uvicorn==0.22.0 pyjwt==2.7.0 From 977bfa08b72d6b33306f1154f567831f65e50c30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 03:42:51 +0000 Subject: [PATCH 46/57] Bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.6 to 1.8.7. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.6...v1.8.7) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0393b5cb9..11821deb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -460,7 +460,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@v1.8.7 if: (github.event_name == 'release') with: user: __token__ @@ -468,7 +468,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@v1.8.7 if: (github.event_name == 'release') with: user: __token__ From 092e30a159d3f96e24d6d41321315d83755b2571 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Jul 2023 21:22:03 +0200 Subject: [PATCH 47/57] Attempt CI without brew update --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11821deb2..8ceac4a7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,7 +160,8 @@ jobs: - name: Installation - macOS if: runner.os == 'macOS' run: | - brew update + # brew update + # TODO: Should be the brew upgrade # homebrew fails to update python due to unlinking failures # https://github.com/actions/runner-images/issues/6817 rm /usr/local/bin/2to3 || true From 7ba459db88e2feeb4904844080680eda7d915b4f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 08:45:06 +0200 Subject: [PATCH 48/57] Version bump to 2023.6 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 8f7717dd2..7a0a675d3 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.6.dev' +__version__ = '2023.6' if 'dev' in __version__: from pathlib import Path From 942f0b4fbd18be47880ca503ae1dbb66cf075806 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 6 Jul 2023 20:27:49 +0200 Subject: [PATCH 49/57] Move format_ms_time to datetime_helpers --- freqtrade/data/history/history_utils.py | 2 +- freqtrade/misc.py | 9 --------- freqtrade/plugins/pairlist/VolumePairList.py | 3 +-- freqtrade/util/__init__.py | 7 ++++--- freqtrade/util/datetime_helpers.py | 8 ++++++++ tests/test_misc.py | 16 +--------------- tests/utils/test_datetime_helpers.py | 17 +++++++++++++++-- 7 files changed, 30 insertions(+), 32 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index e61f59cfa..bd925d901 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -14,8 +14,8 @@ from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange -from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist +from freqtrade.util import format_ms_time from freqtrade.util.binance_mig import migrate_binance_futures_data diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 1e84bba87..350ac5eef 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,7 +3,6 @@ Various tool function for Freqtrade and scripts """ import gzip import logging -from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterator, List, Mapping, Optional, TextIO, Union from urllib.parse import urlparse @@ -123,14 +122,6 @@ def pair_to_filename(pair: str) -> str: return pair -def format_ms_time(date: int) -> str: - """ - convert MS date to readable format. - : epoch-string in ms - """ - return datetime.fromtimestamp(date / 1000.0).strftime('%Y-%m-%dT%H:%M:%S') - - def deep_merge_dicts(source, destination, allow_null_overrides: bool = True): """ Values from Source override destination, destination is returned (and modified!!) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 0d5e33847..9e4a4fca9 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -13,9 +13,8 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers -from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter -from freqtrade.util import dt_now +from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) diff --git a/freqtrade/util/__init__.py b/freqtrade/util/__init__.py index bed65a54b..92c79b899 100644 --- a/freqtrade/util/__init__.py +++ b/freqtrade/util/__init__.py @@ -1,5 +1,5 @@ from freqtrade.util.datetime_helpers import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, - dt_utc, shorten_date) + dt_utc, format_ms_time, shorten_date) from freqtrade.util.ft_precise import FtPrecise from freqtrade.util.periodic_cache import PeriodicCache @@ -7,11 +7,12 @@ from freqtrade.util.periodic_cache import PeriodicCache __all__ = [ 'dt_floor_day', 'dt_from_ts', + 'dt_humanize', 'dt_now', 'dt_ts', 'dt_utc', - 'dt_humanize', - 'shorten_date', + 'format_ms_time', 'FtPrecise', 'PeriodicCache', + 'shorten_date', ] diff --git a/freqtrade/util/datetime_helpers.py b/freqtrade/util/datetime_helpers.py index 39d134e11..7f44cbdb0 100644 --- a/freqtrade/util/datetime_helpers.py +++ b/freqtrade/util/datetime_helpers.py @@ -61,3 +61,11 @@ def dt_humanize(dt: datetime, **kwargs) -> str: :param kwargs: kwargs to pass to arrow's humanize() """ return arrow.get(dt).humanize(**kwargs) + + +def format_ms_time(date: int) -> str: + """ + convert MS date to readable format. + : epoch-string in ms + """ + return datetime.fromtimestamp(date / 1000.0).strftime('%Y-%m-%dT%H:%M:%S') diff --git a/tests/test_misc.py b/tests/test_misc.py index 03a236d73..21c832c2c 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1,6 +1,5 @@ # pragma pylint: disable=missing-docstring,C0103 -import datetime from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock @@ -9,7 +8,7 @@ import pandas as pd import pytest from freqtrade.misc import (dataframe_to_json, decimals_per_coin, deep_merge_dicts, file_dump_json, - file_load_json, format_ms_time, json_to_dataframe, pair_to_filename, + file_load_json, json_to_dataframe, pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, safe_value_fallback2) @@ -91,19 +90,6 @@ def test_pair_to_filename(pair, expected_result): assert pair_s == expected_result -def test_format_ms_time() -> None: - # Date 2018-04-10 18:02:01 - date_in_epoch_ms = 1523383321000 - date = format_ms_time(date_in_epoch_ms) - assert type(date) is str - res = datetime.datetime(2018, 4, 10, 18, 2, 1, tzinfo=datetime.timezone.utc) - assert date == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') - res = datetime.datetime(2017, 12, 13, 8, 2, 1, tzinfo=datetime.timezone.utc) - # Date 2017-12-13 08:02:01 - date_in_epoch_ms = 1513152121000 - assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') - - def test_safe_value_fallback(): dict1 = {'keya': None, 'keyb': 2, 'keyc': 5, 'keyd': None} assert safe_value_fallback(dict1, 'keya', 'keyb') == 2 diff --git a/tests/utils/test_datetime_helpers.py b/tests/utils/test_datetime_helpers.py index 5aec0da54..222410027 100644 --- a/tests/utils/test_datetime_helpers.py +++ b/tests/utils/test_datetime_helpers.py @@ -3,8 +3,8 @@ from datetime import datetime, timedelta, timezone import pytest import time_machine -from freqtrade.util import dt_floor_day, dt_from_ts, dt_now, dt_ts, dt_utc, shorten_date -from freqtrade.util.datetime_helpers import dt_humanize +from freqtrade.util import (dt_floor_day, dt_from_ts, dt_humanize, dt_now, dt_ts, dt_utc, + format_ms_time, shorten_date) def test_dt_now(): @@ -57,3 +57,16 @@ def test_dt_humanize() -> None: assert dt_humanize(dt_now()) == 'just now' assert dt_humanize(dt_now(), only_distance=True) == 'instantly' assert dt_humanize(dt_now() - timedelta(hours=16), only_distance=True) == '16 hours' + + +def test_format_ms_time() -> None: + # Date 2018-04-10 18:02:01 + date_in_epoch_ms = 1523383321000 + date = format_ms_time(date_in_epoch_ms) + assert type(date) is str + res = datetime(2018, 4, 10, 18, 2, 1, tzinfo=timezone.utc) + assert date == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') + res = datetime(2017, 12, 13, 8, 2, 1, tzinfo=timezone.utc) + # Date 2017-12-13 08:02:01 + date_in_epoch_ms = 1513152121000 + assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') From e734a664b4082f08560868fd12a9ffa1b4dae42c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 08:58:10 +0200 Subject: [PATCH 50/57] bump develop-version to 2023.7.dev --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 8f7717dd2..2c8dde56d 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.6.dev' +__version__ = '2023.7.dev' if 'dev' in __version__: from pathlib import Path From f64b9503f9373f56d219442a79ad88f38b363d8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 09:08:32 +0200 Subject: [PATCH 51/57] scipy 1.11 doesn't support python 3.8 any longer --- requirements-hyperopt.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 920da615c..aafbad608 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,8 @@ -r requirements.txt # Required for hyperopt -scipy==1.11.1 +scipy==1.11.1; python_version >= '3.9' +scipy==1.10.1; python_version < '3.9' scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.12.2 From 98ba0042d8a7fdd8d9e9512d2784a8a375b8eb16 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Fri, 7 Jul 2023 09:27:09 +0200 Subject: [PATCH 52/57] Bump datasieve 0.1.7 --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index e8e3b9334..4a25a86ff 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -9,4 +9,4 @@ catboost==1.2; 'arm' not in platform_machine lightgbm==3.3.5 xgboost==1.7.6 tensorboard==2.13.0 -datasieve==0.1.6 +datasieve==0.1.7 From 9b447cdf1e8a6b86b767c2f7ae04357baf808c5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 09:37:14 +0200 Subject: [PATCH 53/57] Bump pandas to 2.0.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebb1c9a16..af2b5712b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.24.3 -pandas==2.0.2 +pandas==2.0.3 pandas-ta==0.3.14b ccxt==3.1.44 From c93a27af7d0e680cf11ae22377e068895171884a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 08:22:16 +0000 Subject: [PATCH 54/57] Bump ccxt from 3.1.44 to 4.0.12 Bumps [ccxt](https://github.com/ccxt/ccxt) from 3.1.44 to 4.0.12. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/3.1.44...4.0.12) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af2b5712b..615c00d2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.3 pandas==2.0.3 pandas-ta==0.3.14b -ccxt==3.1.44 +ccxt==4.0.12 cryptography==41.0.1; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 From 01db789d429e60196dcb271e32d821506437fe73 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 10:56:35 +0200 Subject: [PATCH 55/57] Improve release documentation --- docs/developer.md | 8 ++++++- docs/includes/release_template.md | 37 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docs/includes/release_template.md diff --git a/docs/developer.md b/docs/developer.md index 2782f0117..4784e5352 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -453,7 +453,13 @@ Once the PR against stable is merged (best right after merging): * Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. * Use "stable" as reference (this step comes after the above PR is merged). -* Use the above changelog as release comment (as codeblock) +* Use the above changelog as release comment (as codeblock). +* Use the below snippet for the new release + +??? Tip "Release template" + ```` + --8<-- "includes/release_template.md" + ```` ## Releases diff --git a/docs/includes/release_template.md b/docs/includes/release_template.md new file mode 100644 index 000000000..87a3564da --- /dev/null +++ b/docs/includes/release_template.md @@ -0,0 +1,37 @@ +## Highlighted changes + +- ... + +### How to update + +As always, you can update your bot using one of the following commands: + +#### docker-compose + +```bash +docker-compose pull +docker-compose up -d +``` + +#### Installation via setup script + +``` +# Deactivate venv and run +./setup.sh --update +``` + +#### Plain native installation + +``` +git pull +pip install -U -r requirements.txt +``` + +
+Expand full changelog + +``` + +``` + +
From 65550335eefc2fd5df596d397454e0a54686b01a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 11:15:15 +0200 Subject: [PATCH 56/57] Add explicit online test for get_trade_history part of #8860 --- tests/exchange/test_ccxt_compat.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index b1a6f1c65..c1967abcd 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -43,6 +43,7 @@ EXCHANGES = { 'hasQuoteVolumeFutures': True, 'leverage_tiers_public': False, 'leverage_in_spot_market': False, + 'trades_lookback_hours': 4, 'private_methods': [ 'fapiPrivateGetPositionSideDual', 'fapiPrivateGetMultiAssetsMargin' @@ -98,6 +99,7 @@ EXCHANGES = { 'timeframe': '1h', 'leverage_tiers_public': False, 'leverage_in_spot_market': True, + 'trades_lookback_hours': 12, }, 'kucoin': { 'pair': 'XRP/USDT', @@ -640,7 +642,21 @@ class TestCCXTExchange: assert isinstance(funding_fee, float) # assert funding_fee > 0 - # TODO: tests fetch_trades (?) + def test_ccxt__async_get_trade_history(self, exchange: EXCHANGE_FIXTURE_TYPE): + exch, exchangename = exchange + if not (lookback := EXCHANGES[exchangename].get('trades_lookback_hours')): + pytest.skip('test_fetch_trades not enabled for this exchange') + pair = EXCHANGES[exchangename]['pair'] + since = int((datetime.now(timezone.utc) - timedelta(hours=lookback)).timestamp() * 1000) + res = exch.loop.run_until_complete( + exch._async_get_trade_history(pair, since, None, None) + ) + assert len(res) == 2 + res_pair, res_trades = res + assert res_pair == pair + assert isinstance(res_trades, list) + assert res_trades[0][0] >= since + assert len(res_trades) > 1200 def test_ccxt_get_fee(self, exchange: EXCHANGE_FIXTURE_TYPE): exch, exchangename = exchange From a9e239ca7a16c1a32555e36cc0e5a6afbafd7d63 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Jul 2023 11:23:34 +0200 Subject: [PATCH 57/57] Don't use future date for downloading new trade data closes #8860 --- freqtrade/data/history/history_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index bd925d901..cca2cfa72 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -354,7 +354,7 @@ def _download_trades_history(exchange: Exchange, trades = [] if not since: - since = int((datetime.now() - timedelta(days=-new_pairs_days)).timestamp()) * 1000 + since = int((datetime.now() - timedelta(days=new_pairs_days)).timestamp()) * 1000 from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: