From 8b5e6bf9be6c5d2897bb9672304bd306095046a1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 4 Dec 2023 16:23:55 +0100 Subject: [PATCH 01/10] chore: fix pandas warning about merging on datetime. fix pandas warning about inconsistent key lists in concat --- freqtrade/freqai/data_drawer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index f8b8f0de4..afd70c300 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -284,6 +284,10 @@ class FreqaiDataDrawer: new_pred["date_pred"] = dataframe["date"] hist_preds = self.historic_predictions[pair].copy() + # ensure both dataframes have the same date format so they can be merged + new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"], utc=True) + hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"], utc=True) + # find the closest common date between new_pred and historic predictions # and cut off the new_pred dataframe at that date common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner") @@ -294,7 +298,9 @@ class FreqaiDataDrawer: "predictions. You likely left your FreqAI instance offline " f"for more than {len(dataframe.index)} candles.") - df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + # reindex new_pred columns to match the historic predictions dataframe + new_pred_reindexed = new_pred.reindex(columns=hist_preds.columns) + df_concat = pd.concat([hist_preds, new_pred_reindexed], ignore_index=True) # any missing values will get zeroed out so users can see the exact # downtime in FreqUI From caac77c90b54bdd65021c64f6cf9aba89f620ef8 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Tue, 5 Dec 2023 12:16:08 +0100 Subject: [PATCH 02/10] Update data_drawer.py --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index afd70c300..e3027267b 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -285,8 +285,8 @@ class FreqaiDataDrawer: hist_preds = self.historic_predictions[pair].copy() # ensure both dataframes have the same date format so they can be merged - new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"], utc=True) - hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"], utc=True) + new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"]) + hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"]) # find the closest common date between new_pred and historic predictions # and cut off the new_pred dataframe at that date From 46c81d70184efcb0ffa54e6267ecc7651d7dbbf3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 17:04:16 +0100 Subject: [PATCH 03/10] Revert "Bump tables from 3.9.1 to 3.9.2" --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8e40eeec8..5ef69111d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ technical==1.4.0 tabulate==0.9.0 pycoingecko==3.1.0 jinja2==3.1.2 -tables==3.9.2 +tables==3.9.1 joblib==1.3.2 rich==13.7.0 pyarrow==14.0.1; platform_machine != 'armv7l' From 6ee792069de39b0f9ae02ccb7ccc1d161b8ea455 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:34 +0100 Subject: [PATCH 04/10] Remove bitrex-specific section --- docs/faq.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 50aaa03a3..196bd4308 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -128,12 +128,6 @@ This warning can point to one of the below problems: * Barely traded pair -> Check the pair on the exchange webpage, look at the timeframe your strategy uses. If the pair does not have any volume in some candles (usually visualized with a "volume 0" bar, and a "_" as candle), this pair did not have any trades in this timeframe. These pairs should ideally be avoided, as they can cause problems with order-filling. * API problem -> API returns wrong data (this only here for completeness, and should not happen with supported exchanges). -### I'm getting the "RESTRICTED_MARKET" message in the log - -Currently known to happen for US Bittrex users. - -Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. - ### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io). From ef042ae5ec23790a1ea9338bb38f777f78da97cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:48 +0100 Subject: [PATCH 05/10] Remove exchange tests on bittrex --- tests/exchange/test_exchange.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e932fa9bb..1d26b706f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -24,7 +24,7 @@ from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_pat # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] +EXCHANGES = ['binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] get_entry_rate_data = [ ('other', 20, 19, 10, 0.0, 20), # Full ask side @@ -3901,11 +3901,11 @@ def test_set_margin_mode(mocker, default_conf, margin_mode): ("kraken", TradingMode.SPOT, None, False), ("kraken", TradingMode.MARGIN, MarginMode.ISOLATED, True), ("kraken", TradingMode.FUTURES, MarginMode.ISOLATED, True), - ("bittrex", TradingMode.SPOT, None, False), - ("bittrex", TradingMode.MARGIN, MarginMode.CROSS, True), - ("bittrex", TradingMode.MARGIN, MarginMode.ISOLATED, True), - ("bittrex", TradingMode.FUTURES, MarginMode.CROSS, True), - ("bittrex", TradingMode.FUTURES, MarginMode.ISOLATED, True), + ("bitmart", TradingMode.SPOT, None, False), + ("bitmart", TradingMode.MARGIN, MarginMode.CROSS, True), + ("bitmart", TradingMode.MARGIN, MarginMode.ISOLATED, True), + ("bitmart", TradingMode.FUTURES, MarginMode.CROSS, True), + ("bitmart", TradingMode.FUTURES, MarginMode.ISOLATED, True), ("gate", TradingMode.MARGIN, MarginMode.ISOLATED, True), ("okx", TradingMode.SPOT, None, False), ("okx", TradingMode.MARGIN, MarginMode.CROSS, True), From 4061eaf888c7c78f9d5916f31deea6c6ee731725 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:55 +0100 Subject: [PATCH 06/10] Remove online tests for bittrex --- tests/exchange_online/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index 35c9a9d85..875faeded 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -14,14 +14,6 @@ EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str] # Exchanges that should be tested online EXCHANGES = { - 'bittrex': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': False, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - }, 'binance': { 'pair': 'BTC/USDT', 'stake_currency': 'USDT', From 3739a10735d9a0b7d854a81dab4f50894c280b7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:10:09 +0000 Subject: [PATCH 07/10] Bump nbconvert from 7.11.0 to 7.12.0 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.11.0 to 7.12.0. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md) - [Commits](https://github.com/jupyter/nbconvert/compare/v7.11.0...v7.12.0) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d81e46f29..2302a0426 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ isort==5.12.0 time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.11.0 +nbconvert==7.12.0 # mypy types types-cachetools==5.3.0.7 From 3b4b833dd070120c50a967066cba0c5fd7d34a83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Dec 2023 18:22:28 +0100 Subject: [PATCH 08/10] Remove unused mock --- tests/rpc/test_rpc_telegram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b07951c0e..5d190dcc6 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -109,7 +109,6 @@ def get_telegram_testobject(mocker, default_conf, mock=True, ftbot=None): _start_thread=MagicMock(), ) if not ftbot: - mocker.patch('freqtrade.exchange.exchange.Exchange._init_async_loop') ftbot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(ftbot) telegram = Telegram(rpc, default_conf) From 71178ff1d2b3334dca9fa433677e2d0a4f9f9248 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Dec 2023 19:48:19 +0100 Subject: [PATCH 09/10] Remove "activate" activator they raise warnings as they're unused. --- freqtrade/freqai/prediction_models/XGBoostRFRegressor.py | 2 +- freqtrade/freqai/prediction_models/XGBoostRegressor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py index f43585ab0..1949ad536 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py @@ -45,7 +45,7 @@ class XGBoostRFRegressor(BaseRegressionModel): model = XGBRFRegressor(**self.model_training_parameters) - model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard) + model.set_params(callbacks=[TBCallback(dk.data_path)]) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) # set the callbacks to empty so that we can serialize to disk later diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index f8b4d353d..f1a2474da 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -45,7 +45,7 @@ class XGBoostRegressor(BaseRegressionModel): model = XGBRegressor(**self.model_training_parameters) - model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard) + model.set_params(callbacks=[TBCallback(dk.data_path)]) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) # set the callbacks to empty so that we can serialize to disk later From b14873400d57c4ce616b0b5b4cb2c24e1edee258 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Dec 2023 07:15:05 +0100 Subject: [PATCH 10/10] Fix odd import in freqai tests --- tests/freqai/test_freqai_datakitchen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 8d09cfc58..cac9d9838 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -10,9 +10,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from tests.conftest import get_patched_exchange -from tests.freqai.conftest import (get_patched_data_kitchen, get_patched_freqai_strategy, +from tests.freqai.conftest import (get_patched_data_kitchen, get_patched_freqai_strategy, is_mac, make_unfiltered_dataframe) -from tests.freqai.test_freqai_interface import is_mac @pytest.mark.parametrize(