From 628963c20704ca7d47f4d2ec79cfaaf048fc8c82 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 12 Sep 2023 12:19:12 +0200 Subject: [PATCH 01/50] chore: fix bug associated with leaving FreqAI offline for more than 1 candle. --- freqtrade/freqai/data_drawer.py | 50 ++++++++++++++++---- freqtrade/freqai/freqai_interface.py | 2 +- freqtrade/templates/FreqaiExampleStrategy.py | 2 +- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b6ded83b1..49c673adc 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -263,23 +263,55 @@ class FreqaiDataDrawer: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() return - def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: + def set_initial_return_values(self, pair: str, + pred_df: DataFrame, + dataframe: DataFrame + ) -> None: """ Set the initial return values to the historical predictions dataframe. This avoids needing to repredict on historical candles, and also stores historical predictions despite retrainings (so stored predictions are true predictions, not just inferencing on trained - data) + data). + + We also aim to keep the date from historical predictions so that the FreqUI displays + zeros during any downtime (between FreqAI reloads). """ - hist_df = self.historic_predictions - len_diff = len(hist_df[pair].index) - len(pred_df.index) - if len_diff < 0: - df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]], - ignore_index=True, keys=hist_df[pair].keys()) + new_pred = pred_df.copy() + # set new_pred values to nans (we want to signal to user that there was nothing + # historically made during downtime. The newest pred will get appeneded later in + # append_model_predictions) + new_pred.iloc[:, :] = np.nan + new_pred["date"] = dataframe["date"] + + hist_preds = self.historic_predictions[pair].copy() + # rename date_pred column to date so that we can merge on date + hist_preds = hist_preds.rename(columns={"date_pred": "date"}) + + # find the closest common date between new_pred and historic predictions + # and cut off the new_pred dataframe at that date + common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner") + if len(common_dates.index) > 0: + new_pred = new_pred.iloc[len(common_dates):] else: - df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True) + logger.error("No common dates found between new predictions and historic predictions. " + "You likely left your FreqAI instance offline for more than " + f"{len(dataframe.index)} candles.") + + df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + + # remove last row because we will append that later in append_model_predictions() + df_concat = df_concat.iloc[:-1] + # any missing values will get zeroed out so users can see the exact + # downtime in FreqUI df_concat = df_concat.fillna(0) - self.model_return_values[pair] = df_concat + + # rename date column back to date_pred + df_concat = df_concat.rename(columns={"date": "date_pred"}) + + self.historic_predictions[pair] = df_concat + + self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: NDArray[np.int_], diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index efae6d060..f64a3b8f0 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -453,7 +453,7 @@ class IFreqaiModel(ABC): pred_df, do_preds = self.predict(dataframe, dk) if pair not in self.dd.historic_predictions: self.set_initial_historic_predictions(pred_df, dk, pair, dataframe) - self.dd.set_initial_return_values(pair, pred_df) + self.dd.set_initial_return_values(pair, pred_df, dataframe) dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe) return diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 084cf2e89..e64570b9e 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -31,7 +31,7 @@ class FreqaiExampleStrategy(IStrategy): plot_config = { "main_plot": {}, "subplots": { - "&-s_close": {"prediction": {"color": "blue"}}, + "&-s_close": {"&-s_close": {"color": "blue"}}, "do_predict": { "do_predict": {"color": "brown"}, }, From 844ab4aef5d5f7dc986e9f83d517df9883695625 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 14 Sep 2023 00:05:59 +0200 Subject: [PATCH 02/50] chore: add tests for set_initial_return_values --- freqtrade/freqai/data_drawer.py | 6 +- freqtrade/freqai/freqai_interface.py | 6 +- tests/freqai/test_freqai_datadrawer.py | 120 +++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 49c673adc..013300dfe 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -294,9 +294,9 @@ class FreqaiDataDrawer: if len(common_dates.index) > 0: new_pred = new_pred.iloc[len(common_dates):] else: - logger.error("No common dates found between new predictions and historic predictions. " - "You likely left your FreqAI instance offline for more than " - f"{len(dataframe.index)} candles.") + logger.warning("No common dates found between new predictions and historic " + "predictions. You likely left your FreqAI instance offline " + f"for more than {len(dataframe.index)} candles.") df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index f64a3b8f0..33d23aa73 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -645,11 +645,11 @@ class IFreqaiModel(ABC): If the user reuses an identifier on a subsequent instance, this function will not be called. In that case, "real" predictions will be appended to the loaded set of historic predictions. - :param df: DataFrame = the dataframe containing the training feature data - :param model: Any = A model which was `fit` using a common library such as - catboost or lightgbm + :param pred_df: DataFrame = the dataframe containing the predictions coming + out of a model :param dk: FreqaiDataKitchen = object containing methods for data analysis :param pair: str = current pair + :param strat_df: DataFrame = dataframe coming from strategy """ self.dd.historic_predictions[pair] = pred_df diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 8ab2c75da..b7414884f 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -1,7 +1,9 @@ import shutil from pathlib import Path +from unittest.mock import patch +import pandas as pd import pytest from freqtrade.configuration import TimeRange @@ -135,3 +137,121 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co match=r'Historic predictions not found.*' ): freqai.dd.get_timerange_from_live_historic_predictions() + + +class MockClass: # This represents your class that has the set_initial_return_values method. + def __init__(self): + self.historic_predictions = {} + self.model_return_values = {} + + # ... set_initial_return_values function here ... + + +def test_set_initial_return_values(mocker, freqai_conf): + """ + Simple test of the set initial return values that ensures + we are concatening and ffilling values properly. + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-08-30" + end_x_plus_5 = "2023-09-03" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert (hist_pred_df['date_pred'].iloc[-1] == + pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)) + assert 'date' not in hist_pred_df.columns + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == + hist_pred_df.tail(len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_not_called() + + +def test_set_initial_return_values_warning(mocker, freqai_conf): + """ + Simple test of set_initial_return_values that hits the warning + associated with leaving a FreqAI bot offline so long that the + exchange candles have no common date with the historic predictions + """ + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + freqai = strategy.freqai + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqai_conf) + # Setup + pair = "BTC/USD" + end_x = "2023-08-31" + start_x_plus_1 = "2023-09-01" + end_x_plus_5 = "2023-09-05" + + historic_data = { + 'date_pred': pd.date_range(end=end_x, periods=5), + 'value': range(1, 6) + } + new_data = { + 'date': pd.date_range(start=start_x_plus_1, end=end_x_plus_5), + 'value': range(6, 11) + } + + freqai.dd.historic_predictions[pair] = pd.DataFrame(historic_data) + + new_pred_df = pd.DataFrame(new_data) + dataframe = pd.DataFrame(new_data) + + # Action + with patch('logging.Logger.warning') as mock_logger_warning: + freqai.dd.set_initial_return_values(pair, new_pred_df, dataframe) + + # Assertions + hist_pred_df = freqai.dd.historic_predictions[pair] + model_return_df = freqai.dd.model_return_values[pair] + + assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1) + assert 'date' not in hist_pred_df.columns + assert 'date_pred' in hist_pred_df.columns + assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros + + # compare values in model_return_df with hist_pred_df + assert (model_return_df["value"].values == hist_pred_df.tail( + len(dataframe))["value"].values).all() + assert model_return_df.shape[0] == len(dataframe) + + # Ensure logger error is not called + mock_logger_warning.assert_called() From c0a600858f9d1fd04c3adcc51f235812e06c3207 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 14 Sep 2023 08:50:06 +0900 Subject: [PATCH 03/50] Change the cache of rangestability to 1 day --- docs/includes/pairlists.md | 4 ++-- freqtrade/plugins/pairlist/rangestabilityfilter.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 9cdcc9bca..74746a96b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -376,7 +376,7 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from "lookback_days": 10, "min_rate_of_change": 0.01, "max_rate_of_change": 0.99, - "refresh_period": 1440 + "refresh_period": 86400 } ] ``` @@ -431,7 +431,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01, - "refresh_period": 1440 + "refresh_period": 86400 }, { "method": "VolatilityFilter", diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f294b882b..f4625f572 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -30,7 +30,7 @@ class RangeStabilityFilter(IPairList): self._days = pairlistconfig.get('lookback_days', 10) self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._max_rate_of_change = pairlistconfig.get('max_rate_of_change') - self._refresh_period = pairlistconfig.get('refresh_period', 1440) + self._refresh_period = pairlistconfig.get('refresh_period', 86400) self._def_candletype = self._config['candle_type_def'] self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) From 310c9f60341ccf017af629e07c2bb50a4e3faabc Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Thu, 14 Sep 2023 13:49:31 +0200 Subject: [PATCH 04/50] Update test_freqai_datadrawer.py --- tests/freqai/test_freqai_datadrawer.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index b7414884f..b511f3c7b 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -139,14 +139,6 @@ def test_get_timerange_from_backtesting_live_df_pred_not_found(mocker, freqai_co freqai.dd.get_timerange_from_live_historic_predictions() -class MockClass: # This represents your class that has the set_initial_return_values method. - def __init__(self): - self.historic_predictions = {} - self.model_return_values = {} - - # ... set_initial_return_values function here ... - - def test_set_initial_return_values(mocker, freqai_conf): """ Simple test of the set initial return values that ensures From 454c2343a861d45f82f644bd29ad11ad9061d77d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 14 Sep 2023 18:27:41 +0200 Subject: [PATCH 05/50] More clarity for adjust_trade_position callback docstring --- docs/strategy-callbacks.md | 7 ++++--- freqtrade/strategy/interface.py | 7 ++++--- .../strategy_methods_advanced.j2 | 11 ++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 5ce898dbb..6c86595d3 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -823,7 +823,7 @@ class DigDeeperStrategy(IStrategy): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -832,8 +832,9 @@ class DigDeeperStrategy(IStrategy): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 43d2a0baf..94c78f3c7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -513,7 +513,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -522,8 +522,9 @@ class IStrategy(ABC, HyperStrategyMixin): :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. diff --git a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 index 95c6df2ea..4e1875084 100644 --- a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 @@ -257,7 +257,7 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, """ Custom trade adjustment logic, returning the stake amount that a trade should be increased or decreased. - This means extra buy or sell orders with additional fees. + This means extra entry or exit orders with additional fees. Only called when `position_adjustment_enable` is set to True. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ @@ -266,8 +266,9 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param trade: trade object. :param current_time: datetime object, containing the current datetime - :param current_rate: Current buy rate. - :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param current_rate: Current entry rate (same as current_entry_profit) + :param current_profit: Current profit (as ratio), calculated based on current_rate + (same as current_entry_profit). :param min_stake: Minimal stake size allowed by exchange (for both entries and exits) :param max_stake: Maximum stake allowed (either through balance, or by exchange limits). :param current_entry_rate: Current rate using entry pricing. @@ -276,8 +277,8 @@ def adjust_trade_position(self, trade: 'Trade', current_time: datetime, :param current_exit_profit: Current profit using exit pricing. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade, - Positive values to increase position, Negative values to decrease position. - Return None for no action. + Positive values to increase position, Negative values to decrease position. + Return None for no action. """ return None From 33bf7f9f30b60db9026a8784ab96e7dfe23588db Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 14 Sep 2023 20:05:01 +0200 Subject: [PATCH 06/50] Ensure test doesn't fail based on daytime failed if it's run in the first hour of the (UTC) day --- tests/rpc/test_rpc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 49700b7f4..0a0cc5b33 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -263,7 +263,11 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert isnan(fiat_profit_sum) -def test__rpc_timeunit_profit(default_conf_usdt, ticker, fee, markets, mocker) -> None: +def test__rpc_timeunit_profit( + default_conf_usdt, ticker, fee, markets, mocker, time_machine) -> None: + + time_machine.move_to("2023-09-05 10:00:00 +00:00", tick=False) + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( EXMS, From 3919bf3740ae951ec0f1ec273667f685d8360f3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 15 Sep 2023 07:15:16 +0200 Subject: [PATCH 07/50] Adjust sequence of trade json output --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 59717abcb..679c160ad 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -627,8 +627,8 @@ class LocalTrade: 'amount_precision': self.amount_precision, 'price_precision': self.price_precision, 'precision_mode': self.precision_mode, - 'orders': orders_json, 'has_open_orders': self.has_open_orders, + 'orders': orders_json, } @staticmethod From cf96ad1d1bb3f81aef1197e252fb056d8c96f2c7 Mon Sep 17 00:00:00 2001 From: axel Date: Sat, 16 Sep 2023 02:32:03 -0400 Subject: [PATCH 08/50] add trade param to custom entry price in interface, bot, backtesting, exemples --- docs/strategy-callbacks.md | 2 +- docs/strategy_migration.md | 4 ++-- freqtrade/freqtradebot.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/strategy/interface.py | 3 ++- tests/strategy/strats/strategy_test_v3_custom_entry_price.py | 4 +++- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 6c86595d3..a818315ce 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -520,7 +520,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index d4d5f0068..aaa655473 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -271,7 +271,7 @@ New string argument `side` - which can be either `"long"` or `"short"`. ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, entry_tag: Optional[str], **kwargs) -> float: return proposed_rate ``` @@ -280,7 +280,7 @@ After: ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return proposed_rate ``` diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c895aa5c9..a4a938cac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -937,7 +937,7 @@ class FreqtradeBot(LoggingMixin): # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( - pair=pair, current_time=datetime.now(timezone.utc), + pair=pair, trade=trade, current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b267b315..8f7ec4ab2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -738,7 +738,7 @@ class Backtesting: if order_type == 'limit': new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=propose_rate)( - pair=pair, current_time=current_time, + pair=pair, trade=trade, current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 94c78f3c7..2c3dd5790 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -395,7 +395,8 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Trade, current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. diff --git a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py index 872984156..b24782f82 100644 --- a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py +++ b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py @@ -6,6 +6,8 @@ from typing import Optional from pandas import DataFrame from strategy_test_v3 import StrategyTestV3 +from freqtrade.persistence import Trade + class StrategyTestV3CustomEntryPrice(StrategyTestV3): """ @@ -31,7 +33,7 @@ class StrategyTestV3CustomEntryPrice(StrategyTestV3): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return self.new_entry_price From 8378a0234d6e42f58e9c15d84d038252ef7461f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 09:10:56 +0200 Subject: [PATCH 09/50] Filter exit_order_count on canceled orders only --- freqtrade/constants.py | 3 ++- freqtrade/persistence/trade_model.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 318c414e1..265cfd5d5 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -77,7 +77,8 @@ DL_DATA_TIMEFRAMES = ['1m', '5m'] ENV_VAR_PREFIX = 'FREQTRADE__' -NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') +CANCELED_EXCHANGE_STATES = ('cancelled', 'canceled', 'expired') +NON_OPEN_EXCHANGE_STATES = CANCELED_EXCHANGE_STATES + ('closed',) # Define decimals per coin for outputs # Only used for outputs. diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 679c160ad..9d727ab75 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -13,8 +13,9 @@ from sqlalchemy import (Enum, Float, ForeignKey, Integer, ScalarResult, Select, from sqlalchemy.orm import Mapped, lazyload, mapped_column, relationship, validates from typing_extensions import Self -from freqtrade.constants import (CUSTOM_TAG_MAX_LENGTH, DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, - NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) +from freqtrade.constants import (CANCELED_EXCHANGE_STATES, CUSTOM_TAG_MAX_LENGTH, + DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, + BuySell, LongShort) from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precision, @@ -827,7 +828,8 @@ class LocalTrade: Get amount of failed exiting orders assumes full exits. """ - return len([o for o in self.orders if o.ft_order_side == self.exit_side]) + return len([o for o in self.orders if o.ft_order_side == self.exit_side + and o.status in CANCELED_EXCHANGE_STATES]) def _calc_open_trade_value(self, amount: float, open_rate: float) -> float: """ From ae4021da142a429c90e6324a4f2cd7c1c670caf6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 09:11:31 +0200 Subject: [PATCH 10/50] Rename get_canceled exit orders ... --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence/trade_model.py | 2 +- tests/persistence/test_persistence.py | 4 ++-- tests/test_freqtradebot.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c895aa5c9..ae15cdab1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1362,7 +1362,7 @@ class FreqtradeBot(LoggingMixin): self.handle_cancel_enter(trade, order, order_id, reason) else: canceled = self.handle_cancel_exit(trade, order, order_id, reason) - canceled_count = trade.get_exit_order_count() + canceled_count = trade.get_canceled_exit_order_count() max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: logger.warning(f'Emergency exiting trade {trade}, as the exit order ' diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 9d727ab75..2d2e4a93e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -823,7 +823,7 @@ class LocalTrade: def update_order(self, order: Dict) -> None: Order.update_orders(self.orders, order) - def get_exit_order_count(self) -> int: + def get_canceled_exit_order_count(self) -> int: """ Get amount of failed exiting orders assumes full exits. diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index ccb71b781..517effbfc 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -1928,11 +1928,11 @@ def test_get_best_pair_lev(fee): @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize('is_short', [True, False]) -def test_get_exit_order_count(fee, is_short): +def test_get_canceled_exit_order_count(fee, is_short): create_mock_trades(fee, is_short=is_short) trade = Trade.get_trades([Trade.pair == 'ETC/BTC']).first() - assert trade.get_exit_order_count() == 1 + assert trade.get_canceled_exit_order_count() == 1 @pytest.mark.usefixtures("init_persistence") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2f3a9a1cf..24caf2ce1 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3052,7 +3052,7 @@ def test_manage_open_orders_exit_usercustom( # 2nd canceled trade - Fail execute exit caplog.clear() - mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) + mocker.patch('freqtrade.persistence.Trade.get_canceled_exit_order_count', return_value=1) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', side_effect=DependencyException) freqtrade.manage_open_orders() From fed24c1308d3427d3d7dae531ae46854cd50a8ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 09:13:20 +0200 Subject: [PATCH 11/50] Improve test for get_canceled_exit_order --- tests/persistence/test_persistence.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 517effbfc..e4493d755 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -1932,6 +1932,10 @@ def test_get_canceled_exit_order_count(fee, is_short): create_mock_trades(fee, is_short=is_short) trade = Trade.get_trades([Trade.pair == 'ETC/BTC']).first() + # No canceled order. + assert trade.get_canceled_exit_order_count() == 0 + + trade.orders[-1].status = 'canceled' assert trade.get_canceled_exit_order_count() == 1 From a7d6efdcd66f423e0488a04c69037bf97de987a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 09:23:03 +0200 Subject: [PATCH 12/50] Fix order amounts in test --- tests/test_freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 24caf2ce1..f5aa57ba3 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2991,6 +2991,8 @@ def test_manage_open_orders_exit_usercustom( is_short, open_trade_usdt, caplog ) -> None: default_conf_usdt["unfilledtimeout"] = {"entry": 1440, "exit": 1440, "exit_timeout_count": 1} + limit_sell_order_old['amount'] = open_trade_usdt.amount + limit_sell_order_old['remaining'] = open_trade_usdt.amount if is_short: limit_sell_order_old['side'] = 'buy' From 3a7f390510d753d52860b15255fc202f52435cf0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 09:30:55 +0200 Subject: [PATCH 13/50] Cancel based forceexits shouldn't trigger a full exit. --- freqtrade/freqtradebot.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ae15cdab1..217e3b6bb 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1364,16 +1364,19 @@ class FreqtradeBot(LoggingMixin): canceled = self.handle_cancel_exit(trade, order, order_id, reason) canceled_count = trade.get_canceled_exit_order_count() max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) - if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: - logger.warning(f'Emergency exiting trade {trade}, as the exit order ' - f'timed out {max_timeouts} times.') - self.emergency_exit(trade, order['price']) + if (canceled and max_timeouts > 0 and canceled_count >= max_timeouts): + logger.warning(f"Emergency exiting trade {trade}, as the exit order " + f"timed out {max_timeouts} times. force selling {order['amount']}.") + self.emergency_exit(trade, order['price'], order['amount']) - def emergency_exit(self, trade: Trade, price: float) -> None: + def emergency_exit( + self, trade: Trade, price: float, sub_trade_amt: Optional[float] = None) -> None: try: self.execute_trade_exit( trade, price, - exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT)) + exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT), + sub_trade_amt=sub_trade_amt + ) except DependencyException as exception: logger.warning( f'Unable to emergency exit trade {trade.pair}: {exception}') From 91c710408a15c995716429b2d1937d0d365b8ccf Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sat, 16 Sep 2023 03:36:36 -0400 Subject: [PATCH 14/50] fix flake8, set trade object param as Optional --- freqtrade/strategy/interface.py | 2 +- tests/strategy/strats/strategy_test_v3_custom_entry_price.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 2c3dd5790..8830293e9 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -395,7 +395,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, trade: Trade, current_time: datetime, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ diff --git a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py index b24782f82..04e044977 100644 --- a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py +++ b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py @@ -33,7 +33,8 @@ class StrategyTestV3CustomEntryPrice(StrategyTestV3): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe - def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Trade, current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return self.new_entry_price From 394d758d3203c7917ca558cc2842d107374b2363 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 14:01:27 +0200 Subject: [PATCH 15/50] Add missing fields to json (and json parse) --- freqtrade/persistence/trade_model.py | 5 +++++ tests/persistence/test_persistence.py | 4 ++++ tests/persistence/test_trade_fromjson.py | 7 +++++++ tests/rpc/test_rpc.py | 1 + 4 files changed, 17 insertions(+) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 2d2e4a93e..4ab869e85 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -628,6 +628,7 @@ class LocalTrade: 'amount_precision': self.amount_precision, 'price_precision': self.price_precision, 'precision_mode': self.precision_mode, + 'contract_size': self.contract_size, 'has_open_orders': self.has_open_orders, 'orders': orders_json, } @@ -1788,6 +1789,10 @@ class Trade(ModelBase, LocalTrade): is_short=data["is_short"], trading_mode=data["trading_mode"], funding_fees=data["funding_fees"], + amount_precision=data.get('amount_precision', None), + price_precision=data.get('price_precision', None), + precision_mode=data.get('precision_mode', None), + contract_size=data.get('contract_size', None), ) for order in data["orders"]: diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index e4493d755..396d60c18 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -1385,6 +1385,7 @@ def test_to_json(fee): precision_mode=1, amount_precision=8.0, price_precision=7.0, + contract_size=1, ) result = trade.to_json() assert isinstance(result, dict) @@ -1450,6 +1451,7 @@ def test_to_json(fee): 'amount_precision': 8.0, 'price_precision': 7.0, 'precision_mode': 1, + 'contract_size': 1, 'orders': [], 'has_open_orders': False, } @@ -1471,6 +1473,7 @@ def test_to_json(fee): precision_mode=2, amount_precision=7.0, price_precision=8.0, + contract_size=1 ) result = trade.to_json() assert isinstance(result, dict) @@ -1536,6 +1539,7 @@ def test_to_json(fee): 'amount_precision': 7.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, 'orders': [], 'has_open_orders': False, } diff --git a/tests/persistence/test_trade_fromjson.py b/tests/persistence/test_trade_fromjson.py index 24522e744..24a693c75 100644 --- a/tests/persistence/test_trade_fromjson.py +++ b/tests/persistence/test_trade_fromjson.py @@ -66,6 +66,10 @@ def test_trade_fromjson(): "is_short": false, "trading_mode": "spot", "funding_fees": 0.0, + "amount_precision": 1.0, + "price_precision": 3.0, + "precision_mode": 2, + "contract_size": 1.0, "open_order_id": null, "orders": [ { @@ -180,6 +184,9 @@ def test_trade_fromjson(): assert isinstance(trade.open_date, datetime) assert trade.exit_reason == 'no longer good' assert trade.realized_profit == 2.76315361 + assert trade.precision_mode == 2 + assert trade.amount_precision == 1.0 + assert trade.contract_size == 1.0 assert len(trade.orders) == 5 last_o = trade.orders[-1] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 0a0cc5b33..b8eb51a91 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -90,6 +90,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'amount_precision': 8.0, 'price_precision': 8.0, 'precision_mode': 2, + 'contract_size': 1, 'has_open_orders': False, 'orders': [{ 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, From d01bc0fb9fd1eb2e0009935af5647255845d1232 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 14:35:07 +0200 Subject: [PATCH 16/50] Enforce kwargs for update_trade_state --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 217e3b6bb..39d063cc2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1851,7 +1851,7 @@ class FreqtradeBot(LoggingMixin): def update_trade_state( self, trade: Trade, order_id: Optional[str], - action_order: Optional[Dict[str, Any]] = None, + action_order: Optional[Dict[str, Any]] = None, *, stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary From 6d2d5f93d0b50d1d8791124fecff36f8698986cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 15:04:38 +0200 Subject: [PATCH 17/50] Add exit order test showing behavior in #9186 --- tests/test_freqtradebot.py | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f5aa57ba3..bfe0ec558 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5700,6 +5700,77 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor assert trade.exit_reason == ExitType.SOLD_ON_EXCHANGE.value +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_onexchange_order_exit(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mock_uts = mocker.spy(freqtrade, 'update_trade_state') + + entry_order = limit_order[entry_side(is_short)] + add_entry_order = deepcopy(entry_order) + add_entry_order.update({ + 'id': '_partial_entry_id', + 'amount': add_entry_order['amount'] / 1.5, + 'cost': add_entry_order['cost'] / 1.5, + 'filled': add_entry_order['filled'] / 1.5, + }) + + exit_order_part = deepcopy(limit_order[exit_side(is_short)]) + exit_order_part.update({ + 'id': 'some_random_partial_id', + 'amount': exit_order_part['amount'] / 2, + 'cost': exit_order_part['cost'] / 2, + 'filled': exit_order_part['filled'] / 2, + }) + exit_order = limit_order[exit_side(is_short)] + + # Orders intentionally in the wrong sequence + mock_fo = mocker.patch(f'{EXMS}.fetch_orders', return_value=[ + entry_order, + exit_order_part, + exit_order, + add_entry_order, + ]) + + trade = Trade( + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + is_open=True, + ) + + trade.orders = [ + Order.parse_from_ccxt_object(entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order_part, trade.pair, exit_side(is_short)), + Order.parse_from_ccxt_object(add_entry_order, trade.pair, entry_side(is_short)), + Order.parse_from_ccxt_object(exit_order, trade.pair, exit_side(is_short)), + ] + trade.recalc_trade_from_orders() + Trade.session.add(trade) + Trade.commit() + + freqtrade.handle_onexchange_order(trade) + # assert log_has_re(r"Found previously unknown order .*", caplog) + # Update trade state is called three times, once for every order + assert mock_uts.call_count == 4 + assert mock_fo.call_count == 1 + + trade = Trade.session.scalars(select(Trade)).first() + + assert len(trade.orders) == 4 + assert trade.is_open is True + assert trade.exit_reason is None + assert trade.amount == 5.0 + + def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 3d858f6599eed2e2ff92c580e46712f46aeacb06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 14:43:23 +0200 Subject: [PATCH 18/50] Fix bug closing trades while recalculating closes #9186 --- freqtrade/freqtradebot.py | 2 +- freqtrade/persistence/trade_model.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 39d063cc2..61e6608ca 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1888,7 +1888,7 @@ class FreqtradeBot(LoggingMixin): self.handle_order_fee(trade, order_obj, order) - trade.update_trade(order_obj) + trade.update_trade(order_obj, not send_msg) trade = self._update_trade_after_fill(trade, order_obj) Trade.commit() diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 4ab869e85..8efce76d1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -728,7 +728,7 @@ class LocalTrade: f"Trailing stoploss saved us: " f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.") - def update_trade(self, order: Order) -> None: + def update_trade(self, order: Order, recalculating: bool = False) -> None: """ Updates this entity with amount and actual open/close rates. :param order: order retrieved by exchange.fetch_order() @@ -770,8 +770,9 @@ class LocalTrade: self.precision_mode, self.contract_size) if ( isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC) - or order.safe_amount_after_fee > amount_tr + or (not recalculating and order.safe_amount_after_fee > amount_tr) ): + # When recalculating a trade, only comming out to 0 can force a close self.close(order.safe_price) else: self.recalc_trade_from_orders() From 5d8e0573f6bf941653821af556e38cd936e2c6e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 14:56:37 +0200 Subject: [PATCH 19/50] Run handle_onexchange_order test without dry-run --- tests/test_freqtradebot.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index bfe0ec558..0886c9fca 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5660,6 +5660,7 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_short, caplog): + default_conf_usdt['dry_run'] = False freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) mock_uts = mocker.spy(freqtrade, 'update_trade_state') @@ -5671,17 +5672,17 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor ]) trade = Trade( - pair='ETH/USDT', - fee_open=0.001, - fee_close=0.001, - open_rate=entry_order['price'], - open_date=dt_now(), - stake_amount=entry_order['cost'], - amount=entry_order['amount'], - exchange="binance", - is_short=is_short, - leverage=1, - ) + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=dt_now(), + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + ) trade.orders.append(Order.parse_from_ccxt_object( entry_order, 'ADA/USDT', entry_side(is_short)) From c90be601f5e0485360abb52c563eb154653d477c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 17:48:31 +0200 Subject: [PATCH 20/50] Add offset to "fetch_orders" to avoid missing the initial order --- freqtrade/freqtradebot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 61e6608ca..bd6353016 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -456,7 +456,8 @@ class FreqtradeBot(LoggingMixin): Only used balance disappeared, which would make exiting impossible. """ try: - orders = self.exchange.fetch_orders(trade.pair, trade.open_date_utc) + orders = self.exchange.fetch_orders( + trade.pair, trade.open_date_utc - timedelta(seconds=10)) prev_exit_reason = trade.exit_reason prev_trade_state = trade.is_open for order in orders: From 220bc3c23eb16e09f0eee08bdd18b8da9068f783 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 17:56:16 +0200 Subject: [PATCH 21/50] Rename fetch_orders_emulate to make it non-protected --- freqtrade/exchange/exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ddb00ecef..fac02509b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1420,7 +1420,7 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - def __fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: orders = [] if self.exchange_has('fetchClosedOrders'): orders = self._api.fetch_closed_orders(pair, since=since_ms) @@ -1450,9 +1450,9 @@ class Exchange: except ccxt.NotSupported: # Some exchanges don't support fetchOrders # attempt to fetch open and closed orders separately - orders = self.__fetch_orders_emulate(pair, since_ms) + orders = self._fetch_orders_emulate(pair, since_ms) else: - orders = self.__fetch_orders_emulate(pair, since_ms) + orders = self._fetch_orders_emulate(pair, since_ms) self._log_exchange_response('fetch_orders', orders) orders = [self._order_contracts_to_amount(o) for o in orders] return orders From 1d0f1bd1ee37f00a8750b6ff5ca4a36c6f32bb70 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sat, 16 Sep 2023 13:13:23 -0400 Subject: [PATCH 22/50] update doc --- docs/strategy-callbacks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index a818315ce..321874f6f 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -510,6 +510,8 @@ Each of these methods are called right before placing an order on the exchange. !!! Note If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration. +!!! Note + Within this function you will have access to the Trade object, giving you the flexibility to adjust your current entry or exit price in relation to the current trade status. This object is available as soon as the first entry order associated with the trade is created. ### Custom order entry and exit price example ``` python From 3f4715ba496599fd1dc603135ff795639fb46c12 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sat, 16 Sep 2023 13:20:10 -0400 Subject: [PATCH 23/50] update custom_entry_price docstring --- freqtrade/strategy/interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8830293e9..82e9f5609 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -406,6 +406,7 @@ class IStrategy(ABC, HyperStrategyMixin): When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object. :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. From 659168d341b473dbcdab0c95f8fce7fb080adccd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 19:43:20 +0200 Subject: [PATCH 24/50] Improve cancel_stop test --- tests/exchange/test_exchange.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 73baa15f2..99540bd9f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3122,25 +3122,28 @@ def test_cancel_stoploss_order(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): default_conf['dry_run'] = False + mock_prefix = 'freqtrade.exchange.gate.Gate' + if exchange_name == 'okx': + mock_prefix = 'freqtrade.exchange.okx.Okx' mocker.patch(f'{EXMS}.fetch_stoploss_order', return_value={'for': 123}) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', return_value={'for': 123}) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', return_value={'for': 123}) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) res = {'fee': {}, 'status': 'canceled', 'amount': 1234} mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value=res) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value=res) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value=res) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == res mocker.patch(f'{EXMS}.cancel_stoploss_order', return_value='canceled') - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', return_value='canceled') + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', return_value='canceled') # Fall back to fetch_stoploss_order co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co == {'for': 123} exc = InvalidOrderException("") mocker.patch(f'{EXMS}.fetch_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.fetch_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.fetch_stoploss_order', side_effect=exc) co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) assert co['amount'] == 555 assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} @@ -3148,7 +3151,7 @@ def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): with pytest.raises(InvalidOrderException): exc = InvalidOrderException("Did not find order") mocker.patch(f'{EXMS}.cancel_stoploss_order', side_effect=exc) - mocker.patch('freqtrade.exchange.gate.Gate.cancel_stoploss_order', side_effect=exc) + mocker.patch(f'{mock_prefix}.cancel_stoploss_order', side_effect=exc) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=123) From afb1b787c894eb7794dbeab8ca2222aa13acb739 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 19:43:05 +0200 Subject: [PATCH 25/50] add okx to fully tested exchanges --- freqtrade/exchange/okx.py | 2 +- tests/exchange/test_exchange.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c0629240d..c9f7c52cb 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -186,7 +186,7 @@ class Okx(Exchange): def _convert_stop_order(self, pair: str, order_id: str, order: Dict) -> Dict: if ( - order['status'] == 'closed' + order.get('status', 'open') == 'closed' and (real_order_id := order.get('info', {}).get('ordId')) is not None ): # Once a order triggered, we fetch the regular followup order. diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 99540bd9f..66cd55010 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -24,7 +24,7 @@ from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_pat # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit'] +EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] get_entry_rate_data = [ ('other', 20, 19, 10, 0.0, 20), # Full ask side @@ -1312,8 +1312,11 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice, leverage=3.0 ) - assert exchange._set_leverage.call_count == 1 - assert exchange.set_margin_mode.call_count == 1 + if exchange_name != 'okx': + assert exchange._set_leverage.call_count == 1 + assert exchange.set_margin_mode.call_count == 1 + else: + assert api_mock.set_leverage.call_count == 1 assert order['amount'] == 0.01 @@ -2044,7 +2047,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ ) # Required candles candles = (end_ts - start_ts) / 300_000 - exp = candles // exchange.ohlcv_candle_limit('5m', CandleType.SPOT) + 1 + exp = candles // exchange.ohlcv_candle_limit('5m', candle_type, start_ts) + 1 # Depending on the exchange, this should be called between 1 and 6 times. assert exchange._api_async.fetch_ohlcv.call_count == exp @@ -3226,8 +3229,14 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name): api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value={'id': '123', 'symbol': 'TKN/BTC'}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == {'id': '123', 'symbol': 'TKN/BTC'} + res = {'id': '123', 'symbol': 'TKN/BTC'} + if exchange_name == 'okx': + res = {'id': '123', 'symbol': 'TKN/BTC', 'type': 'stoploss'} + assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == res + if exchange_name == 'okx': + # Tested separately. + return with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) @@ -3547,6 +3556,8 @@ def test_get_markets_error(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): + if exchange_name == 'okx': + pytest.skip("Tested separately for okx") exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) timeframes = ('1m', '5m', '1h') expected = exchange._ft_has['ohlcv_candle_limit'] From 5b857aeaf048daa343149b8ecf56d51b2cd0bbe2 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sat, 16 Sep 2023 13:46:56 -0400 Subject: [PATCH 26/50] fix custom_entry_price trade object type test --- freqtrade/strategy/interface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 82e9f5609..a1101875f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -16,7 +16,7 @@ from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirecti from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds from freqtrade.misc import remove_entry_exit_signals -from freqtrade.persistence import Order, PairLocks, Trade +from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, _create_and_merge_informative_pair, @@ -395,8 +395,8 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, - proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Union[Trade, LocalTrade, None], + current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. From a7cd9d77f202f3f535d465d98f315ea9ec8a1e70 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sat, 16 Sep 2023 13:58:59 -0400 Subject: [PATCH 27/50] ignore custom_entry_price trade object type test, remove LocalTrade as type --- freqtrade/freqtradebot.py | 4 +++- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/strategy/interface.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a4a938cac..0d145db3b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -937,7 +937,9 @@ class FreqtradeBot(LoggingMixin): # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( - pair=pair, trade=trade, current_time=datetime.now(timezone.utc), + pair=pair, + trade=trade, # type: ignore[arg-type] + current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8f7ec4ab2..6db996589 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -738,7 +738,9 @@ class Backtesting: if order_type == 'limit': new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=propose_rate)( - pair=pair, trade=trade, current_time=current_time, + pair=pair, + trade=trade, # type: ignore[arg-type] + current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a1101875f..0e0a29a45 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -16,7 +16,7 @@ from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirecti from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds from freqtrade.misc import remove_entry_exit_signals -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, _create_and_merge_informative_pair, @@ -395,7 +395,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, trade: Union[Trade, LocalTrade, None], + def custom_entry_price(self, pair: str, trade: Trade, current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ From 5745722a37ca36101fa0d866469b3fa4eececceb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 19:52:30 +0200 Subject: [PATCH 28/50] Increase time for fetch_orders in test --- tests/exchange/test_exchange.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 66cd55010..8826fb876 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1680,7 +1680,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) mocker.patch(f'{EXMS}.exchange_has', return_value=True) - start_time = datetime.now(timezone.utc) - timedelta(days=5) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + expected = 1 + if exchange_name == 'bybit': + expected = 3 exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # Not available in dry-run @@ -1690,10 +1693,10 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) res = exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected assert api_mock.fetch_open_orders.call_count == 0 assert api_mock.fetch_closed_orders.call_count == 0 - assert len(res) == 2 + assert len(res) == 2 * expected res = exchange.fetch_orders('mocked', start_time) @@ -1707,13 +1710,17 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): if endpoint == 'fetchOpenOrders': return True + if exchange_name == 'okx': + # Special OKX case is tested separately + return + mocker.patch(f'{EXMS}.exchange_has', has_resp) # happy path without fetchOrders - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) assert api_mock.fetch_orders.call_count == 0 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected mocker.patch(f'{EXMS}.exchange_has', return_value=True) @@ -1726,11 +1733,11 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): api_mock.fetch_open_orders.reset_mock() api_mock.fetch_closed_orders.reset_mock() - res = exchange.fetch_orders('mocked', start_time) + exchange.fetch_orders('mocked', start_time) - assert api_mock.fetch_orders.call_count == 1 - assert api_mock.fetch_open_orders.call_count == 1 - assert api_mock.fetch_closed_orders.call_count == 1 + assert api_mock.fetch_orders.call_count == expected + assert api_mock.fetch_open_orders.call_count == expected + assert api_mock.fetch_closed_orders.call_count == expected def test_fetch_trading_fees(default_conf, mocker): From 14c5f435aa68bcdad12dd75a64893cae56dcf319 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Sep 2023 20:00:39 +0200 Subject: [PATCH 29/50] okx: Use proper history endpoint for fetch_orders --- freqtrade/exchange/okx.py | 17 ++++++++++ tests/exchange/test_okx.py | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index c9f7c52cb..b99911994 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -1,4 +1,5 @@ import logging +from datetime import timedelta from typing import Any, Dict, List, Optional, Tuple import ccxt @@ -10,6 +11,7 @@ from freqtrade.exceptions import (DDosProtection, OperationalException, Retryabl from freqtrade.exchange import Exchange, date_minus_candles from freqtrade.exchange.common import retrier from freqtrade.misc import safe_value_fallback2 +from freqtrade.util import dt_now, dt_ts logger = logging.getLogger(__name__) @@ -240,3 +242,18 @@ class Okx(Exchange): pair=pair, params=params1, ) + + def _fetch_orders_emulate(self, pair: str, since_ms: int) -> List[Dict]: + orders = [] + + orders = self._api.fetch_closed_orders(pair, since=since_ms) + if (since_ms < dt_ts(dt_now() - timedelta(days=6, hours=23))): + # Regular fetch_closed_orders only returns 7 days of data. + # Force usage of "archive" endpoint, which returns 3 months of data. + params = {'method': 'privateGetTradeOrdersHistoryArchive'} + orders_hist = self._api.fetch_closed_orders(pair, since=since_ms, params=params) + orders.extend(orders_hist) + + orders_open = self._api.fetch_open_orders(pair, since=since_ms) + orders.extend(orders_open) + return orders diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index e8f059118..736c630e0 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -618,3 +618,70 @@ def test__get_stop_params_okx(mocker, default_conf): assert params['tdMode'] == 'isolated' assert params['posSide'] == 'net' + + +def test_fetch_orders_okx(default_conf, mocker, limit_order): + + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[ + limit_order['buy'], + limit_order['sell'], + ]) + api_mock.fetch_open_orders = MagicMock(return_value=[limit_order['buy']]) + api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + start_time = datetime.now(timezone.utc) - timedelta(days=20) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + # Not available in dry-run + assert exchange.fetch_orders('mocked', start_time) == [] + assert api_mock.fetch_orders.call_count == 0 + default_conf['dry_run'] = False + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='okx') + + def has_resp(_, endpoint): + if endpoint == 'fetchOrders': + return False + if endpoint == 'fetchClosedOrders': + return True + if endpoint == 'fetchOpenOrders': + return True + + mocker.patch(f'{EXMS}.exchange_has', has_resp) + + history_params = {'method': 'privateGetTradeOrdersHistoryArchive'} + + # happy path without fetchOrders + exchange.fetch_orders('mocked', start_time) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params + + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + # regular closed_orders endpoint only has history for 7 days. + exchange.fetch_orders('mocked', datetime.now(timezone.utc) - timedelta(days=6)) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 1 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + + # Unhappy path - first fetch-orders call fails. + api_mock.fetch_orders = MagicMock(side_effect=ccxt.NotSupported()) + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + exchange.fetch_orders('mocked', start_time) + + assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 2 + assert 'params' not in api_mock.fetch_closed_orders.call_args_list[0][1] + assert api_mock.fetch_closed_orders.call_args_list[1][1]['params'] == history_params From d81fdeb3ed4c1f6c4d7ae5873b1b2411db31421b Mon Sep 17 00:00:00 2001 From: Axel CHERUBIN Date: Sun, 17 Sep 2023 03:06:00 -0400 Subject: [PATCH 30/50] Update docs/strategy-callbacks.md Co-authored-by: Matthias --- docs/strategy-callbacks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 321874f6f..8e7bee5f9 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -522,7 +522,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional['Trade'], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, From 224213840d90994acfae181ee96dfcb9baec3683 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sun, 17 Sep 2023 03:13:40 -0400 Subject: [PATCH 31/50] update trade object as optional parameter --- docs/strategy_migration.md | 4 ++-- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/interface.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/strategy_migration.md b/docs/strategy_migration.md index aaa655473..9e6f56e49 100644 --- a/docs/strategy_migration.md +++ b/docs/strategy_migration.md @@ -271,7 +271,7 @@ New string argument `side` - which can be either `"long"` or `"short"`. ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, entry_tag: Optional[str], **kwargs) -> float: return proposed_rate ``` @@ -280,7 +280,7 @@ After: ``` python hl_lines="3" class AwesomeStrategy(IStrategy): - def custom_entry_price(self, pair: str, trade: 'Trade', current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: return proposed_rate ``` diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0d145db3b..1dd7884e0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -938,7 +938,7 @@ class FreqtradeBot(LoggingMixin): custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( pair=pair, - trade=trade, # type: ignore[arg-type] + trade=trade, current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0e0a29a45..1c9a18b5c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -395,7 +395,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.stoploss - def custom_entry_price(self, pair: str, trade: Trade, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ From 2bc0c4ecd5f8f46080fcf54843184d84221b533e Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sun, 17 Sep 2023 03:17:07 -0400 Subject: [PATCH 32/50] update docstring --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1c9a18b5c..5cdbb6bf6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -406,7 +406,7 @@ class IStrategy(ABC, HyperStrategyMixin): When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed - :param trade: trade object. + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. From 29a5e049b92ff7812ba329b60fef8cb9132413e8 Mon Sep 17 00:00:00 2001 From: Axel-CH Date: Sun, 17 Sep 2023 03:30:03 -0400 Subject: [PATCH 33/50] edit note wording for custom_entry_price --- docs/strategy-callbacks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 8e7bee5f9..d4950cfdd 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -511,7 +511,7 @@ Each of these methods are called right before placing an order on the exchange. If your custom pricing function return None or an invalid value, price will fall back to `proposed_rate`, which is based on the regular pricing configuration. !!! Note - Within this function you will have access to the Trade object, giving you the flexibility to adjust your current entry or exit price in relation to the current trade status. This object is available as soon as the first entry order associated with the trade is created. + Using custom_entry_price, the Trade object will be available as soon as the first entry order associated with the trade is created, for the first entry, `trade` parameter value will be `None`. ### Custom order entry and exit price example ``` python From 1f1abfe7981e605a03a2e306da6411b8e2834ff1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 17 Sep 2023 17:36:01 +0200 Subject: [PATCH 34/50] fix: avoid duplicate date columns interfering with fit_live_predictions() --- freqtrade/freqai/data_drawer.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 013300dfe..0306282c0 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -282,15 +282,12 @@ class FreqaiDataDrawer: # historically made during downtime. The newest pred will get appeneded later in # append_model_predictions) new_pred.iloc[:, :] = np.nan - new_pred["date"] = dataframe["date"] - + new_pred["date_pred"] = dataframe["date"] hist_preds = self.historic_predictions[pair].copy() - # rename date_pred column to date so that we can merge on date - hist_preds = hist_preds.rename(columns={"date_pred": "date"}) # find the closest common date between new_pred and historic predictions # and cut off the new_pred dataframe at that date - common_dates = pd.merge(new_pred, hist_preds, on="date", how="inner") + common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner") if len(common_dates.index) > 0: new_pred = new_pred.iloc[len(common_dates):] else: @@ -299,18 +296,12 @@ class FreqaiDataDrawer: f"for more than {len(dataframe.index)} candles.") df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) - # remove last row because we will append that later in append_model_predictions() df_concat = df_concat.iloc[:-1] # any missing values will get zeroed out so users can see the exact # downtime in FreqUI df_concat = df_concat.fillna(0) - - # rename date column back to date_pred - df_concat = df_concat.rename(columns={"date": "date_pred"}) - self.historic_predictions[pair] = df_concat - self.model_return_values[pair] = df_concat.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions(self, pair: str, predictions: DataFrame, From 158bf097744796a1072dd72a362dbfb3f7b845b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Sep 2023 19:40:44 +0200 Subject: [PATCH 35/50] Fix type checking error --- tests/strategy/strats/strategy_test_v3_custom_entry_price.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py index 04e044977..607ff6e1e 100644 --- a/tests/strategy/strats/strategy_test_v3_custom_entry_price.py +++ b/tests/strategy/strats/strategy_test_v3_custom_entry_price.py @@ -33,7 +33,7 @@ class StrategyTestV3CustomEntryPrice(StrategyTestV3): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return dataframe - def custom_entry_price(self, pair: str, trade: Trade, current_time: datetime, + def custom_entry_price(self, pair: str, trade: Optional[Trade], current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: From f6fce2162c4238bdf279b92dc214566c6129bf7d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Sep 2023 19:57:13 +0200 Subject: [PATCH 36/50] Add new parameter to strategy template --- docs/strategy-callbacks.md | 1 + freqtrade/freqtradebot.py | 3 +-- .../strategy_subtemplates/strategy_methods_advanced.j2 | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index d4950cfdd..e42aa39d2 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -512,6 +512,7 @@ Each of these methods are called right before placing an order on the exchange. !!! Note Using custom_entry_price, the Trade object will be available as soon as the first entry order associated with the trade is created, for the first entry, `trade` parameter value will be `None`. + ### Custom order entry and exit price example ``` python diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1dd7884e0..983d5c826 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -937,8 +937,7 @@ class FreqtradeBot(LoggingMixin): # Don't call custom_entry_price in order-adjust scenario custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, default_retval=enter_limit_requested)( - pair=pair, - trade=trade, + pair=pair, trade=trade, current_time=datetime.now(timezone.utc), proposed_rate=enter_limit_requested, entry_tag=entry_tag, side=trade_side, diff --git a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 index 4e1875084..6fad129c7 100644 --- a/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 @@ -13,7 +13,8 @@ def bot_loop_start(self, current_time: datetime, **kwargs) -> None: """ pass -def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: float, +def custom_entry_price(self, pair: str, trade: Optional['Trade'], + current_time: 'datetime', proposed_rate: float, entry_tag: 'Optional[str]', side: str, **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -23,6 +24,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: When not implemented by a strategy, returns None, orderbook is used to set entry price :param pair: Pair that's currently analyzed + :param trade: trade object (None for initial entries). :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. From 74ed0e0b1164daa3ad67421f0ff2273110e5c67a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:38:34 +0000 Subject: [PATCH 37/50] Bump ruff from 0.0.287 to 0.0.290 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.287 to 0.0.290. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.0.287...v0.0.290) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e779429a0..f19c932fd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.287 +ruff==0.0.290 mypy==1.5.1 pre-commit==3.4.0 pytest==7.4.2 From d185b2020ac0be920534549725042ed3a104e1a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:38:44 +0000 Subject: [PATCH 38/50] Bump filelock from 3.12.3 to 3.12.4 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.12.3 to 3.12.4. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.12.3...3.12.4) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 4cc7cc4c7..06f8ddbaf 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,4 +5,4 @@ scipy==1.11.2 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.12.3 +filelock==3.12.4 From e6d01b04ea0415c18e1c2284eab9a0b0532164de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:38:57 +0000 Subject: [PATCH 39/50] Bump numpy from 1.25.2 to 1.26.0 Bumps [numpy](https://github.com/numpy/numpy) from 1.25.2 to 1.26.0. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.25.2...v1.26.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f91a8d2b5..6cdb3b3f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.25.2 +numpy==1.26.0 pandas==2.0.3 pandas-ta==0.3.14b From 9b35dae4654b4909b545068c81bf4f6fb47b3079 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:39:05 +0000 Subject: [PATCH 40/50] Bump mkdocs-material from 9.2.8 to 9.3.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.2.8 to 9.3.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.2.8...9.3.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index e2292e640..4d66ebb5e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.4.4 mkdocs==1.5.2 -mkdocs-material==9.2.8 +mkdocs-material==9.3.1 mdx_truly_sane_lists==1.3 pymdown-extensions==10.3 jinja2==3.1.2 From 533a16658c22a196f6cfbf365401603331ac2e06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:39:14 +0000 Subject: [PATCH 41/50] Bump rich from 13.5.2 to 13.5.3 Bumps [rich](https://github.com/Textualize/rich) from 13.5.2 to 13.5.3. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.5.2...v13.5.3) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f91a8d2b5..05c6cd163 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ jinja2==3.1.2 tables==3.8.0 blosc==1.11.1 joblib==1.3.2 -rich==13.5.2 +rich==13.5.3 pyarrow==13.0.0; platform_machine != 'armv7l' # find first, C search in arrays From 6e1f457fb3f88f98bf4ea6be08c33c9e0f662a0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:39:41 +0000 Subject: [PATCH 42/50] Bump python-rapidjson from 1.10 to 1.11 Bumps [python-rapidjson](https://github.com/python-rapidjson/python-rapidjson) from 1.10 to 1.11. - [Changelog](https://github.com/python-rapidjson/python-rapidjson/blob/master/CHANGES.rst) - [Commits](https://github.com/python-rapidjson/python-rapidjson/compare/v1.10...v1.11) --- updated-dependencies: - dependency-name: python-rapidjson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f91a8d2b5..c7fbe076a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ pyarrow==13.0.0; platform_machine != 'armv7l' py_find_1st==1.1.5 # Load ticker files 30% faster -python-rapidjson==1.10 +python-rapidjson==1.11 # Properly format api responses orjson==3.9.7 From c7106a6802e85588586aca7309d137c0f6ee7c90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 03:39:45 +0000 Subject: [PATCH 43/50] Bump plotly from 5.16.1 to 5.17.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.16.1 to 5.17.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.16.1...v5.17.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 9a8c596ad..b2ec35539 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.16.1 +plotly==5.17.0 From 3f85f3cce68cf777fb2f31ead2e106ee33ec4d54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Sep 2023 07:08:08 +0200 Subject: [PATCH 44/50] Exclude UP036 for now --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 40c0e2005..c0123e219 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,8 @@ ignore = ["freqtrade/vendor/**"] line-length = 100 extend-exclude = [".env", ".venv"] target-version = "py38" +# Exclude UP036 as it's causing the "exit if < 3.9" to fail. +extend-ignore = [ "UP036" ] extend-select = [ "C90", # mccabe # "N", # pep8-naming From fd4877759c1323a74e49345445cb2eb7bef6a16a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Sep 2023 18:12:39 +0200 Subject: [PATCH 45/50] Update hard min-requirement for python --- freqtrade/main.py | 4 ++-- pyproject.toml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index a10620498..05e5409ad 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -11,8 +11,8 @@ from freqtrade.util.gc_setup import gc_set_threshold # check min. python version -if sys.version_info < (3, 8): # pragma: no cover - sys.exit("Freqtrade requires Python version >= 3.8") +if sys.version_info < (3, 9): # pragma: no cover + sys.exit("Freqtrade requires Python version >= 3.9") from freqtrade import __version__ from freqtrade.commands import Arguments diff --git a/pyproject.toml b/pyproject.toml index c0123e219..cd0c65916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,6 @@ line-length = 100 extend-exclude = [".env", ".venv"] target-version = "py38" # Exclude UP036 as it's causing the "exit if < 3.9" to fail. -extend-ignore = [ "UP036" ] extend-select = [ "C90", # mccabe # "N", # pep8-naming From cfdd9d6be9db5953273df58bc8695307d9b6e636 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:22:38 +0000 Subject: [PATCH 46/50] Bump lightgbm from 4.0.0 to 4.1.0 Bumps [lightgbm](https://github.com/microsoft/LightGBM) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/microsoft/LightGBM/releases) - [Commits](https://github.com/microsoft/LightGBM/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: lightgbm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 8f690f957..1c31a31e1 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -6,7 +6,7 @@ scikit-learn==1.1.3 joblib==1.3.2 catboost==1.2.1; 'arm' not in platform_machine -lightgbm==4.0.0 +lightgbm==4.1.0 xgboost==1.7.6 tensorboard==2.14.0 datasieve==0.1.7 From 2f63b0199d4ca295e88593c16e1be50418ec1cb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 18:12:29 +0000 Subject: [PATCH 47/50] Bump xgboost from 1.7.6 to 2.0.0 Bumps [xgboost](https://github.com/dmlc/xgboost) from 1.7.6 to 2.0.0. - [Release notes](https://github.com/dmlc/xgboost/releases) - [Changelog](https://github.com/dmlc/xgboost/blob/master/NEWS.md) - [Commits](https://github.com/dmlc/xgboost/compare/v1.7.6...v2.0.0) --- updated-dependencies: - dependency-name: xgboost dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 1c31a31e1..0c11f8a09 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -7,6 +7,6 @@ scikit-learn==1.1.3 joblib==1.3.2 catboost==1.2.1; 'arm' not in platform_machine lightgbm==4.1.0 -xgboost==1.7.6 +xgboost==2.0.0 tensorboard==2.14.0 datasieve==0.1.7 From 8f883f2310a05d6401616a3f1ee9a6b12981be6f Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 19 Sep 2023 12:20:44 +0200 Subject: [PATCH 48/50] chore: fix set_initial_hist_preds test --- tests/freqai/test_freqai_datadrawer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index b511f3c7b..ca4749747 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -181,7 +181,6 @@ def test_set_initial_return_values(mocker, freqai_conf): assert (hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1)) - assert 'date' not in hist_pred_df.columns assert 'date_pred' in hist_pred_df.columns assert hist_pred_df.shape[0] == 7 # Total rows: 5 from historic and 2 new zeros @@ -236,7 +235,6 @@ def test_set_initial_return_values_warning(mocker, freqai_conf): model_return_df = freqai.dd.model_return_values[pair] assert hist_pred_df['date_pred'].iloc[-1] == pd.Timestamp(end_x_plus_5) - pd.Timedelta(days=1) - assert 'date' not in hist_pred_df.columns assert 'date_pred' in hist_pred_df.columns assert hist_pred_df.shape[0] == 9 # Total rows: 5 from historic and 4 new zeros From d21f0f40812969e496096407bdc63e86852c8307 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 19 Sep 2023 12:24:44 +0200 Subject: [PATCH 49/50] chore: add guardrails for users who neglect docs --- freqtrade/freqai/data_kitchen.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 7d4bf39ca..3d1ed7849 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -244,6 +244,14 @@ class FreqaiDataKitchen: f"{self.pair}: dropped {len(unfiltered_df) - len(filtered_df)} training points" f" due to NaNs in populated dataset {len(unfiltered_df)}." ) + if len(unfiltered_df) == 0 and not self.live: + raise OperationalException( + f"{self.pair}: all training data dropped due to NaNs. " + "You likely did not download enough training data prior " + "to your backtest timerange. Hint:\n" + "https://www.freqtrade.io/en/stable/freqai-running/" + "#downloading-data-to-cover-the-full-backtest-period" + ) if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live: worst_indicator = str(unfiltered_df.count().idxmin()) logger.warning( From 830f2219c808cd33af790664c4324c7a7d1de28f Mon Sep 17 00:00:00 2001 From: alex <67626131+alexpvpmindustry@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:44:01 -0700 Subject: [PATCH 50/50] Update exchanges.md - Malaysia is a soverign country... parted with Singapore in 1965. - also fix some formating --- docs/exchanges.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index fb3049ba5..63819393e 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -55,7 +55,7 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t ## Binance !!! Warning "Server location and geo-ip restrictions" - Please be aware that binance restrict api access regarding the server country. The currents and non exhaustive countries blocked are United States, Malaysia (Singapour), Ontario (Canada). Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. + Please be aware that Binance restricts API access regarding the server country. The current and non-exhaustive countries blocked are Canada, Malaysia, Netherlands and United States. Please go to [binance terms > b. Eligibility](https://www.binance.com/en/terms) to find up to date list. Binance supports [time_in_force](configuration.md#understand-order_time_in_force).