From 96e5615d1b9f9d67f3cbe088210b06a086858c1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Sep 2023 20:03:08 +0200 Subject: [PATCH 1/6] Update safe_value_fallback to allow empty 2nd keys --- freqtrade/misc.py | 4 ++-- tests/test_misc.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index f8d730fae..cbebf99eb 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -156,7 +156,7 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): +def safe_value_fallback(obj: dict, key1: str, key2: Optional[str] = None, default_value=None): """ Search a value in obj, return this if it's not None. Then search key2 in obj - return that if it's not none - then use default_value. @@ -165,7 +165,7 @@ def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): if key1 in obj and obj[key1] is not None: return obj[key1] else: - if key2 in obj and obj[key2] is not None: + if key2 and key2 in obj and obj[key2] is not None: return obj[key2] return default_value diff --git a/tests/test_misc.py b/tests/test_misc.py index e94e299fd..7de1adbbc 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -121,6 +121,8 @@ def test_safe_value_fallback(): assert safe_value_fallback(dict1, 'keyNo', 'keyNo') is None assert safe_value_fallback(dict1, 'keyNo', 'keyNo', 55) == 55 + assert safe_value_fallback(dict1, 'keyNo', default_value=55) == 55 + assert safe_value_fallback(dict1, 'keyNo', None, default_value=55) == 55 def test_safe_value_fallback2(): From f0819d9df11145763595a90953fb7a4aedca2617 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Sep 2023 19:55:00 +0200 Subject: [PATCH 2/6] Improve "filled" date assignment for order updates --- freqtrade/persistence/trade_model.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 22444bb49..7d85a8c92 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -20,8 +20,9 @@ from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import (ROUND_DOWN, ROUND_UP, amount_to_contract_precision, price_to_precision) from freqtrade.leverage import interest +from freqtrade.misc import safe_value_fallback from freqtrade.persistence.base import ModelBase, SessionType -from freqtrade.util import FtPrecise, dt_now +from freqtrade.util import FtPrecise, dt_from_ts, dt_now, dt_ts logger = logging.getLogger(__name__) @@ -176,7 +177,9 @@ class Order(ModelBase): # (represents the funding fee since the last order) self.funding_fee = self.trade.funding_fees if (order.get('filled', 0.0) or 0.0) > 0 and not self.order_filled_date: - self.order_filled_date = datetime.now(timezone.utc) + self.order_filled_date = dt_from_ts( + safe_value_fallback(order, 'lastTradeTimestamp', default_value=dt_ts()) + ) self.order_update_date = datetime.now(timezone.utc) def to_ccxt_object(self, stopPriceName: str = 'stopPrice') -> Dict[str, Any]: From ee9d2c637a724b0cb47f6144c2a8272254f01b88 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Sep 2023 20:18:42 +0200 Subject: [PATCH 3/6] Improve "order refind" mechanics --- freqtrade/freqtradebot.py | 49 +++++++++++++++++++++++--------------- tests/test_freqtradebot.py | 3 ++- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1a24009b3..1b414b77f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -457,30 +457,41 @@ class FreqtradeBot(LoggingMixin): """ try: orders = self.exchange.fetch_orders(trade.pair, trade.open_date_utc) + prev_exit_reason = trade.exit_reason + prev_trade_state = trade.is_open for order in orders: trade_order = [o for o in trade.orders if o.order_id == order['id']] - if trade_order: - continue - logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") - order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) - order_obj.order_filled_date = datetime.fromtimestamp( - safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, - tz=timezone.utc) - trade.orders.append(order_obj) - prev_exit_reason = trade.exit_reason - trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value - self.update_trade_state(trade, order['id'], order) + if trade_order: + # We knew this order, but didn't have it updated properly + order_obj = trade_order[0] + else: + logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) + order_obj.order_filled_date = datetime.fromtimestamp( + safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, + tz=timezone.utc) + trade.orders.append(order_obj) + Trade.commit() + trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value + + self.update_trade_state(trade, order['id'], order, send_msg=False) logger.info(f"handled order {order['id']}") - if not trade.is_open: - # Trade was just closed - trade.close_date = order_obj.order_filled_date - Trade.commit() - break - else: - trade.exit_reason = prev_exit_reason - Trade.commit() + + # Refresh trade from database + Trade.session.refresh(trade) + if not trade.is_open: + # Trade was just closed + trade.close_date = max([o.order_filled_date for o in trade.orders + if o.order_filled_date]) + self.order_close_notify(trade, order_obj, + order_obj.ft_order_side == 'stoploss', + send_msg=prev_trade_state != trade.is_open) + else: + trade.exit_reason = prev_exit_reason + Trade.commit() except ExchangeError: logger.warning("Error finding onexchange order.") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 886012535..2f3a9a1cf 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5687,7 +5687,8 @@ def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_shor Trade.session.add(trade) freqtrade.handle_onexchange_order(trade) assert log_has_re(r"Found previously unknown order .*", caplog) - assert mock_uts.call_count == 1 + # Update trade state is called twice, once for the known and once for the unknown order. + assert mock_uts.call_count == 2 assert mock_fo.call_count == 1 trade = Trade.session.scalars(select(Trade)).first() From af1054fa703b96f13cf01465ac81455a2912309e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Sep 2023 07:00:55 +0200 Subject: [PATCH 4/6] Avoid re-implementing existing feature --- freqtrade/freqtradebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1b414b77f..c895aa5c9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -484,8 +484,7 @@ class FreqtradeBot(LoggingMixin): Trade.session.refresh(trade) if not trade.is_open: # Trade was just closed - trade.close_date = max([o.order_filled_date for o in trade.orders - if o.order_filled_date]) + trade.close_date = trade.date_last_filled_utc self.order_close_notify(trade, order_obj, order_obj.ft_order_side == 'stoploss', send_msg=prev_trade_state != trade.is_open) From a866b0a35ec8128fc745c6e767dfc7e3055ac4d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Sep 2023 07:01:05 +0200 Subject: [PATCH 5/6] Improve test correctness --- tests/persistence/test_persistence.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 0f0057d1e..ccb71b781 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -600,7 +600,9 @@ def test_calc_open_close_trade_price( @pytest.mark.usefixtures("init_persistence") -def test_trade_close(fee): +def test_trade_close(fee, time_machine): + time_machine.move_to("2022-09-01 05:00:00 +00:00", tick=False) + trade = Trade( pair='ADA/USDT', stake_amount=60.0, @@ -609,7 +611,7 @@ def test_trade_close(fee): is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, - open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=10), + open_date=dt_now() - timedelta(minutes=10), interest_rate=0.0005, exchange='binance', trading_mode=margin, @@ -628,6 +630,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.entry_side, + order_filled_date=trade.open_date, )) trade.orders.append(Order( ft_order_side=trade.exit_side, @@ -642,6 +645,7 @@ def test_trade_close(fee): status="closed", order_type="limit", side=trade.exit_side, + order_filled_date=dt_now(), )) assert trade.close_profit is None assert trade.close_date is None @@ -650,14 +654,15 @@ def test_trade_close(fee): assert trade.is_open is False assert pytest.approx(trade.close_profit) == 0.094513715 assert trade.close_date is not None + assert trade.close_date_utc == dt_now() - new_date = datetime(2020, 2, 2, 15, 6, 1), - assert trade.close_date != new_date + new_date = dt_now() + timedelta(minutes=5) + assert trade.close_date_utc != new_date # Close should NOT update close_date if the trade has been closed already assert trade.is_open is False trade.close_date = new_date trade.close(2.2) - assert trade.close_date == new_date + assert trade.close_date_utc == new_date @pytest.mark.usefixtures("init_persistence") From a52cf42218022996e4476e7582d55f0d12a12193 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Sep 2023 07:01:51 +0200 Subject: [PATCH 6/6] use last order date to fill order. --- freqtrade/persistence/trade_model.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 7d85a8c92..59717abcb 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -433,13 +433,20 @@ class LocalTrade: return self.amount @property - def date_last_filled_utc(self) -> datetime: + def _date_last_filled_utc(self) -> Optional[datetime]: """ Date of the last filled order""" orders = self.select_filled_orders() - if not orders: + if orders: + return max(o.order_filled_utc for o in orders if o.order_filled_utc) + return None + + @property + def date_last_filled_utc(self) -> datetime: + """ Date of the last filled order - or open_date if no orders are filled""" + dt_last_filled = self._date_last_filled_utc + if not dt_last_filled: return self.open_date_utc - return max([self.open_date_utc, - max(o.order_filled_utc for o in orders if o.order_filled_utc)]) + return max([self.open_date_utc, dt_last_filled]) @property def open_date_utc(self): @@ -775,7 +782,7 @@ class LocalTrade: and marks trade as closed """ self.close_rate = rate - self.close_date = self.close_date or datetime.utcnow() + self.close_date = self.close_date or self._date_last_filled_utc or dt_now() self.is_open = False self.exit_order_status = 'closed' self.recalc_trade_from_orders(is_closing=True)