From 531b5727f24c40502ed08d3be989126fa530d2c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 14:33:34 +0200 Subject: [PATCH 01/16] add fetch_orders exchange wrapper --- freqtrade/exchange/exchange.py | 23 +++++++++++++++++++++++ tests/exchange/test_exchange.py | 26 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 9a303426a..822d1074b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1432,6 +1432,29 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @retrier(retries=0) + def fetch_orders(self, pair: str, since: datetime) -> List[Dict]: + """ + Fetch all orders for a pair "since" + :param pair: Pair for the query + :param since: Starting time for the query + """ + if self._config['dry_run'] or not self.exchange_has('fetchOrders'): + return [] + try: + since_ms = int((since.timestamp() - 10) * 1000) + orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms) + self._log_exchange_response('fetch_orders', orders) + orders = [self._order_contracts_to_amount(o) for o in orders] + return orders + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not fetch positions due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + @retrier def fetch_trading_fees(self) -> Dict[str, Any]: """ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b0760944a..0452f70e3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1773,6 +1773,32 @@ def test_fetch_positions(default_conf, mocker, exchange_name): "fetch_positions", "fetch_positions") +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): + + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[ + limit_order['buy'], + limit_order['sell'], + ]) + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + start_time = datetime.now(timezone.utc) - timedelta(days=5) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + # Not available in dry-run + assert exchange.fetch_orders('mocked', start_time) == [] + + default_conf['dry_run'] = False + + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + res = exchange.fetch_orders('mocked', start_time) + assert len(res) == 2 + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, + "fetch_orders", "fetch_orders", retries=1, + pair='mocked', since=start_time) + + def test_fetch_trading_fees(default_conf, mocker): api_mock = MagicMock() tick = { From d14f50f50db83e2250e6737ea5b47883bf7aabf9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 14:39:18 +0200 Subject: [PATCH 02/16] temporary comment fetch_orders logic --- freqtrade/exchange/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 42a7094ba..3a4a940ca 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -84,6 +84,7 @@ EXCHANGE_HAS_OPTIONAL = [ # 'fetchPositions', # Futures trading # 'fetchLeverageTiers', # Futures initialization # 'fetchMarketLeverageTiers', # Futures initialization + # 'fetchOpenOrders', 'fetchClosedOrders', # 'fetchOrders', # Refinding balance... ] From 81633b7c2ee0b30de77c2e62b2e34f753c0f805c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 15:35:53 +0200 Subject: [PATCH 03/16] Add "handle_onexchange_order" functionality --- freqtrade/freqtradebot.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 89f0ac55d..ae569f7c2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -451,6 +451,35 @@ class FreqtradeBot(LoggingMixin): except ExchangeError: logger.warning(f"Error updating {order.order_id}.") + def handle_onexchange_order(self, trade: Trade): + """ + Try refinding a order that is not in the database. + Only used balance disappeared, which would make exiting impossible. + """ + try: + orders = self.exchange.fetch_orders(trade.pair, trade.open_date_utc) + for order in orders: + trade_order = [o for o in trade.orders if o.order_id == order['id']] + if trade_order: + continue + logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") + order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) + order_obj.order_filled_date = datetime.fromtimestamp( + safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, + tz=timezone.utc) + trade.orders.append(order_obj) + # TODO: how do we handle open_order_id ... + Trade.commit() + self.update_trade_state(trade, order['id'], order) + logger.info(f"handled order {order['id']}") + if not trade.is_open: + # Trade was just closed + trade.close_date = order_obj.order_filled_date + Trade.commit() + continue + + except ExchangeError: + logger.warning("Error finding onexchange order") # # BUY / enter positions / open trades logic and methods # @@ -1034,6 +1063,16 @@ class FreqtradeBot(LoggingMixin): """ trades_closed = 0 for trade in trades: + # TODO: get_total currently fails for futures! + wallet_amount = self.wallets.get_total(trade.safe_base_currency) + + if wallet_amount < trade.amount: + # + logger.warning( + f'Not enough {trade.safe_base_currency} in wallet to exit {trade.pair}. ' + f'Amount needed: {trade.amount}, amount available: {wallet_amount}') + self.handle_onexchange_order(trade) + try: try: if (self.strategy.order_types.get('stoploss_on_exchange') and From 95b35e452d9a2043e2b82742adc389afc1bae546 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 17:13:02 +0200 Subject: [PATCH 04/16] Emulate fetch_orders if it ain't supported natively --- freqtrade/exchange/exchange.py | 22 ++++++++++++++++-- tests/exchange/test_exchange.py | 41 ++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 822d1074b..07abb489f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1439,11 +1439,29 @@ class Exchange: :param pair: Pair for the query :param since: Starting time for the query """ - if self._config['dry_run'] or not self.exchange_has('fetchOrders'): + if self._config['dry_run']: return [] + + def fetch_orders_emulate() -> List[Dict]: + orders = [] + if self.exchange_has('fetchClosedOrders'): + orders = self._api.fetch_closed_orders(pair, since=since_ms) + if self.exchange_has('fetchOpenOrders'): + orders_open = self._api.fetch_open_orders(pair, since=since_ms) + orders.extend(orders_open) + return orders + try: since_ms = int((since.timestamp() - 10) * 1000) - orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms) + if self.exchange_has('fetchOrders'): + try: + orders: List[Dict] = self._api.fetch_orders(pair, since=since_ms) + except ccxt.NotSupported: + # Some exchanges don't support fetchOrders + # attempt to fetch open and closed orders separately + orders = fetch_orders_emulate() + else: + orders = fetch_orders_emulate() self._log_exchange_response('fetch_orders', orders) orders = [self._order_contracts_to_amount(o) for o in orders] return orders diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0452f70e3..5994c56e0 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1781,23 +1781,62 @@ def test_fetch_orders(default_conf, mocker, exchange_name, limit_order): limit_order['buy'], limit_order['sell'], ]) + api_mock.fetch_open_orders = MagicMock(return_value=[limit_order['buy']]) + api_mock.fetch_closed_orders = MagicMock(return_value=[limit_order['buy']]) + mocker.patch(f'{EXMS}.exchange_has', return_value=True) start_time = datetime.now(timezone.utc) - timedelta(days=5) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # Not available in dry-run assert exchange.fetch_orders('mocked', start_time) == [] - + assert api_mock.fetch_orders.call_count == 0 default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) res = exchange.fetch_orders('mocked', start_time) + assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == 0 + assert api_mock.fetch_closed_orders.call_count == 0 assert len(res) == 2 + res = exchange.fetch_orders('mocked', start_time) + + api_mock.fetch_orders.reset_mock() + + def has_resp(_, endpoint): + if endpoint == 'fetchOrders': + return False + if endpoint == 'fetchClosedOrders': + return True + if endpoint == 'fetchOpenOrders': + return True + + mocker.patch(f'{EXMS}.exchange_has', has_resp) + + # happy path without fetchOrders + res = exchange.fetch_orders('mocked', start_time) + assert api_mock.fetch_orders.call_count == 0 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 1 + + mocker.patch(f'{EXMS}.exchange_has', return_value=True) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, "fetch_orders", "fetch_orders", retries=1, pair='mocked', since=start_time) + # Unhappy path - first fetch-orders call fails. + api_mock.fetch_orders = MagicMock(side_effect=ccxt.NotSupported()) + api_mock.fetch_open_orders.reset_mock() + api_mock.fetch_closed_orders.reset_mock() + + res = exchange.fetch_orders('mocked', start_time) + + assert api_mock.fetch_orders.call_count == 1 + assert api_mock.fetch_open_orders.call_count == 1 + assert api_mock.fetch_closed_orders.call_count == 1 + def test_fetch_trading_fees(default_conf, mocker): api_mock = MagicMock() From 974cf6c365348515d3d3ca4cdaabd628a4c0f7f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 17:41:59 +0200 Subject: [PATCH 05/16] Move comment to more appropriate spot --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ae569f7c2..5a253e40c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1575,13 +1575,13 @@ class FreqtradeBot(LoggingMixin): # Update wallets to ensure amounts tied up in a stoploss is now free! self.wallets.update() if self.trading_mode == TradingMode.FUTURES: + # A safe exit amount isn't needed for futures, you can just exit/close the position return amount trade_base_currency = self.exchange.get_pair_base_currency(pair) wallet_amount = self.wallets.get_free(trade_base_currency) logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") if wallet_amount >= amount: - # A safe exit amount isn't needed for futures, you can just exit/close the position return amount elif wallet_amount > amount * 0.98: logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.") From 24cab004790a9073bc45ce9f249910a38a8303ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 17:46:58 +0200 Subject: [PATCH 06/16] Extract amount checking to wallets, implement for futures --- freqtrade/freqtradebot.py | 9 +++------ freqtrade/wallets.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5a253e40c..444fe044a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1063,14 +1063,11 @@ class FreqtradeBot(LoggingMixin): """ trades_closed = 0 for trade in trades: - # TODO: get_total currently fails for futures! - wallet_amount = self.wallets.get_total(trade.safe_base_currency) - if wallet_amount < trade.amount: - # + if not self.wallets.check_exit_amount(trade): logger.warning( - f'Not enough {trade.safe_base_currency} in wallet to exit {trade.pair}. ' - f'Amount needed: {trade.amount}, amount available: {wallet_amount}') + f'Not enough {trade.safe_base_currency} in wallet to exit {trade}. ' + 'Trying to recover.') self.handle_onexchange_order(trade) try: diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 6f86398f3..ecac638c6 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -181,6 +181,35 @@ class Wallets: def get_all_positions(self) -> Dict[str, PositionWallet]: return self._positions + def _check_exit_amount(self, trade: Trade) -> bool: + if trade.trading_mode != TradingMode.FUTURES: + # Slightly higher offset than in safe_exit_amount. + wallet_amount: float = self.get_total(trade.safe_base_currency) * 0.981 + else: + # wallet_amount: float = self.wallets.get_free(trade.safe_base_currency) + position = self._positions.get(trade.pair) + if position is None: + # We don't own anything :O + return False + wallet_amount = position.position + + if wallet_amount >= trade.amount: + return True + return False + + def check_exit_amount(self, trade: Trade) -> bool: + """ + Checks if the exit amount is available in the wallet. + :param trade: Trade to check + :return: True if the exit amount is available, False otherwise + """ + if not self._check_exit_amount(trade): + # Update wallets just to make sure + self.update() + return self._check_exit_amount(trade) + + return True + def get_starting_balance(self) -> float: """ Retrieves starting balance - based on either available capital, From f2696c96095ffe20e10e03d3697168ac7cf8bc57 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 18:09:46 +0200 Subject: [PATCH 07/16] Force special exit reason for "recovered" exits --- freqtrade/enums/exittype.py | 1 + freqtrade/freqtradebot.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/enums/exittype.py b/freqtrade/enums/exittype.py index b025230ba..c21b62667 100644 --- a/freqtrade/enums/exittype.py +++ b/freqtrade/enums/exittype.py @@ -15,6 +15,7 @@ class ExitType(Enum): EMERGENCY_EXIT = "emergency_exit" CUSTOM_EXIT = "custom_exit" PARTIAL_EXIT = "partial_exit" + SOLD_ON_EXCHANGE = "sold_on_exchange" NONE = "" def __str__(self): diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 444fe044a..8b877541c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -470,13 +470,17 @@ class FreqtradeBot(LoggingMixin): trade.orders.append(order_obj) # TODO: how do we handle open_order_id ... Trade.commit() + prev_exit_reason = trade.exit_reason + trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value self.update_trade_state(trade, order['id'], order) logger.info(f"handled order {order['id']}") if not trade.is_open: # Trade was just closed trade.close_date = order_obj.order_filled_date Trade.commit() - continue + break + else: + trade.exit_reason = prev_exit_reason except ExchangeError: logger.warning("Error finding onexchange order") From 0c22710ddd7e67dae0c3a11b7bf9e201ef17b71c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 19:30:24 +0200 Subject: [PATCH 08/16] Add API endpoint to force trade reloading --- freqtrade/rpc/api_server/api_v1.py | 11 +++++++++-- freqtrade/rpc/rpc.py | 12 ++++++++++++ tests/rpc/test_rpc_apiserver.py | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 5ee5e36c4..6642f5827 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -44,7 +44,8 @@ logger = logging.getLogger(__name__) # 2.24: Add cancel_open_order endpoint # 2.25: Add several profit values to /status endpoint # 2.26: increase /balance output -API_VERSION = 2.26 +# 2.27: Add /trades//reload endpoint +API_VERSION = 2.27 # Public API, requires no auth. router_public = APIRouter() @@ -127,11 +128,17 @@ def trades_delete(tradeid: int, rpc: RPC = Depends(get_rpc)): @router.delete('/trades/{tradeid}/open-order', response_model=OpenTradeSchema, tags=['trading']) -def cancel_open_order(tradeid: int, rpc: RPC = Depends(get_rpc)): +def trade_cancel_open_order(tradeid: int, rpc: RPC = Depends(get_rpc)): rpc._rpc_cancel_open_order(tradeid) return rpc._rpc_trade_status([tradeid])[0] +@router.get('/trades/{tradeid}/reload', response_model=OpenTradeSchema, tags=['trading']) +def trade_reload(tradeid: int, rpc: RPC = Depends(get_rpc)): + rpc._rpc_reload_trade_from_exchange(tradeid) + return rpc._rpc_trade_status([tradeid])[0] + + # TODO: Missing response model @router.get('/edge', tags=['info']) def edge(rpc: RPC = Depends(get_rpc)): diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 35e08cbc0..a5f6a0a66 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -741,6 +741,18 @@ class RPC: return {'status': 'No more entries will occur from now. Run /reload_config to reset.'} + def _rpc_reload_trade_from_exchange(self, trade_id: str) -> Dict[str, str]: + """ + Handler for reload_trade_from_exchange. + Reloads a trade from it's orders, should manual interaction have happened. + """ + trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() + if not trade: + raise RPCException(f"Could not find trade with id {trade_id}.") + + self._freqtrade.handle_onexchange_order(trade) + return {'status': 'Reloaded from orders from exchange'} + def __exec_force_exit(self, trade: Trade, ordertype: Optional[str], amount: Optional[float] = None) -> None: # Check if there is there is an open order diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8123e4689..51fddbb88 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -740,6 +740,33 @@ def test_api_delete_open_order(botclient, mocker, fee, markets, ticker, is_short assert cancel_mock.call_count == 1 +@pytest.mark.parametrize('is_short', [True, False]) +def test_api_trade_reload_trade(botclient, mocker, fee, markets, ticker, is_short): + ftbot, client = botclient + patch_get_signal(ftbot, enter_long=not is_short, enter_short=is_short) + stoploss_mock = MagicMock() + cancel_mock = MagicMock() + ftbot.handle_onexchange_order = MagicMock() + mocker.patch.multiple( + EXMS, + markets=PropertyMock(return_value=markets), + fetch_ticker=ticker, + cancel_order=cancel_mock, + cancel_stoploss_order=stoploss_mock, + ) + + rc = client_get(client, f"{BASE_URI}/trades/10/reload") + assert_response(rc, 502) + assert 'Could not find trade with id 10.' in rc.json()['error'] + assert ftbot.handle_onexchange_order.call_count == 0 + + create_mock_trades(fee, is_short=is_short) + Trade.commit() + + rc = client_get(client, f"{BASE_URI}/trades/5/reload") + assert ftbot.handle_onexchange_order.call_count == 1 + + def test_api_logs(botclient): ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/logs") From 7287e9da1dce45fd839df640e3f918b194e769d6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 19:34:37 +0200 Subject: [PATCH 09/16] Add telegram endpoint for reload_trade --- freqtrade/rpc/telegram.py | 12 ++++++++++++ tests/rpc/test_rpc_telegram.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6e509950c..0779e5795 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -196,6 +196,7 @@ class Telegram(RPCHandler): self._force_enter, order_side=SignalDirection.LONG)), CommandHandler('forceshort', partial( self._force_enter, order_side=SignalDirection.SHORT)), + CommandHandler('reload_trade', self._reload_trade_from_exchange), CommandHandler('trades', self._trades), CommandHandler('delete', self._delete_trade), CommandHandler(['coo', 'cancel_open_order'], self._cancel_open_order), @@ -1074,6 +1075,17 @@ class Telegram(RPCHandler): msg = self._rpc._rpc_stopentry() await self._send_msg(f"Status: `{msg['status']}`") + @authorized_only + async def _reload_trade_from_exchange(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /reload_trade . + """ + if not context.args or len(context.args) == 0: + raise RPCException("Trade-id not set.") + trade_id = context.args[0] + msg = self._rpc._rpc_reload_trade_from_exchange(trade_id) + await self._send_msg(f"Status: `{msg['status']}`") + @authorized_only async def _force_exit(self, update: Update, context: CallbackContext) -> None: """ diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 4b4c2b028..8570b2ad5 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1760,6 +1760,25 @@ async def test_telegram_delete_trade(mocker, update, default_conf, fee, is_short assert "Please make sure to take care of this asset" in msg_mock.call_args_list[0][0][0] +@pytest.mark.parametrize('is_short', [True, False]) +async def test_telegram_reload_trade_from_exchange(mocker, update, default_conf, fee, is_short): + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + context = MagicMock() + context.args = [] + + await telegram._reload_trade_from_exchange(update=update, context=context) + assert "Trade-id not set." in msg_mock.call_args_list[0][0][0] + + msg_mock.reset_mock() + create_mock_trades(fee, is_short=is_short) + + context.args = [5] + + await telegram._reload_trade_from_exchange(update=update, context=context) + assert "Status: `Reloaded from orders from exchange`" in msg_mock.call_args_list[0][0][0] + + @pytest.mark.parametrize('is_short', [True, False]) async def test_telegram_delete_open_order(mocker, update, default_conf, fee, is_short, ticker): From 25bed7bb8700207d52be30b2632482b49bf0c098 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 19:39:52 +0200 Subject: [PATCH 10/16] Update telegram help with reload_trade --- freqtrade/rpc/telegram.py | 1 + tests/rpc/test_rpc_telegram.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0779e5795..7eb3028ee 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1573,6 +1573,7 @@ class Telegram(RPCHandler): "*/fx |all:* `Alias to /forceexit`\n" f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}" "*/delete :* `Instantly delete the given trade in the database`\n" + "*/reload_trade :* `Relade trade from exchange Orders`\n" "*/cancel_open_order :* `Cancels open orders for trade. " "Only valid when the trade has open orders.`\n" "*/coo |all:* `Alias to /cancel_open_order`\n" diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 8570b2ad5..c5bdb5e5b 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -143,8 +143,8 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " "['balance'], ['start'], ['stop'], " "['forceexit', 'forcesell', 'fx'], ['forcebuy', 'forcelong'], ['forceshort'], " - "['trades'], ['delete'], ['cancel_open_order', 'coo'], ['performance'], " - "['buys', 'entries'], ['exits', 'sells'], ['mix_tags'], " + "['reload_trade'], ['trades'], ['delete'], ['cancel_open_order', 'coo'], " + "['performance'], ['buys', 'entries'], ['exits', 'sells'], ['mix_tags'], " "['stats'], ['daily'], ['weekly'], ['monthly'], " "['count'], ['locks'], ['delete_locks', 'unlock'], " "['reload_conf', 'reload_config'], ['show_conf', 'show_config'], " From d0b5c7d2168a49fe6f9b7c093aa9870398f1c37e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Apr 2023 19:40:05 +0200 Subject: [PATCH 11/16] update telegram/api documentation with new endpoint --- docs/rest-api.md | 4 +++- docs/telegram-usage.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 860a44499..5b33bfa6f 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -134,7 +134,9 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `reload_config` | Reloads the configuration file. | `trades` | List last trades. Limited to 500 trades per call. | `trade/` | Get specific trade. -| `delete_trade ` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `trade/` | DELETE - Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `trade//open-order` | DELETE - Cancel open order for this trade. +| `trade//reload` | GET - Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange. | `show_config` | Shows part of the current configuration with relevant settings to operation. | `logs` | Shows last log messages. | `status` | Lists all open trades. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index e6017e271..1b36c60ad 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -187,6 +187,7 @@ official commands. You can ask at any moment for help with `/help`. | `/forcelong [rate]` | Instantly buys the given pair. Rate is optional and only applies to limit orders. (`force_entry_enable` must be set to True) | `/forceshort [rate]` | Instantly shorts the given pair. Rate is optional and only applies to limit orders. This will only work on non-spot markets. (`force_entry_enable` must be set to True) | `/delete ` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `/reload_trade ` | Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange. | `/cancel_open_order | /coo ` | Cancel an open order for a trade. | **Metrics** | | `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) From b0b036c457f0cc62e0e98ced1df19d68633d4761 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Apr 2023 06:45:09 +0200 Subject: [PATCH 12/16] Fix logic lapsus in check_exit_amount --- freqtrade/freqtradebot.py | 3 +++ freqtrade/wallets.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8b877541c..59f764111 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -463,6 +463,7 @@ class FreqtradeBot(LoggingMixin): if trade_order: continue logger.info(f"Found previously unknown order {order['id']} for {trade.pair}.") + order_obj = Order.parse_from_ccxt_object(order, trade.pair, order['side']) order_obj.order_filled_date = datetime.fromtimestamp( safe_value_fallback(order, 'lastTradeTimestamp', 'timestamp') // 1000, @@ -473,6 +474,7 @@ class FreqtradeBot(LoggingMixin): prev_exit_reason = trade.exit_reason trade.exit_reason = ExitType.SOLD_ON_EXCHANGE.value self.update_trade_state(trade, order['id'], order) + logger.info(f"handled order {order['id']}") if not trade.is_open: # Trade was just closed @@ -481,6 +483,7 @@ class FreqtradeBot(LoggingMixin): break else: trade.exit_reason = prev_exit_reason + Trade.commit() except ExchangeError: logger.warning("Error finding onexchange order") diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index ecac638c6..9a33d1fb1 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -184,7 +184,7 @@ class Wallets: def _check_exit_amount(self, trade: Trade) -> bool: if trade.trading_mode != TradingMode.FUTURES: # Slightly higher offset than in safe_exit_amount. - wallet_amount: float = self.get_total(trade.safe_base_currency) * 0.981 + wallet_amount: float = self.get_total(trade.safe_base_currency) * (2 - 0.981) else: # wallet_amount: float = self.wallets.get_free(trade.safe_base_currency) position = self._positions.get(trade.pair) From d29a425baa81e050e20844e7090460bf8f2fafca Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Apr 2023 07:03:28 +0200 Subject: [PATCH 13/16] Update parameter type in RPC modules --- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a5f6a0a66..9064c8a58 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -741,7 +741,7 @@ class RPC: return {'status': 'No more entries will occur from now. Run /reload_config to reset.'} - def _rpc_reload_trade_from_exchange(self, trade_id: str) -> Dict[str, str]: + def _rpc_reload_trade_from_exchange(self, trade_id: int) -> Dict[str, str]: """ Handler for reload_trade_from_exchange. Reloads a trade from it's orders, should manual interaction have happened. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7eb3028ee..b25aa3e32 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1082,7 +1082,7 @@ class Telegram(RPCHandler): """ if not context.args or len(context.args) == 0: raise RPCException("Trade-id not set.") - trade_id = context.args[0] + trade_id = int(context.args[0]) msg = self._rpc._rpc_reload_trade_from_exchange(trade_id) await self._send_msg(f"Status: `{msg['status']}`") From e88e259033a45b555f00fccfaf8382e86dafd75c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Apr 2023 07:09:26 +0200 Subject: [PATCH 14/16] explicitly test check_exit_amount --- tests/test_wallets.py | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 7ccc8d0f5..09adf6e15 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -3,9 +3,11 @@ from copy import deepcopy from unittest.mock import MagicMock import pytest +from sqlalchemy import select from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import DependencyException +from freqtrade.persistence import Trade from tests.conftest import EXMS, create_mock_trades, get_patched_freqtradebot, patch_wallet @@ -364,3 +366,48 @@ def test_sync_wallet_futures_dry(mocker, default_conf, fee): free = freqtrade.wallets.get_free('BTC') used = freqtrade.wallets.get_used('BTC') assert free + used == total + + +def test_check_exit_amount(mocker, default_conf, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + update_mock = mocker.patch("freqtrade.wallets.Wallets.update") + total_mock = mocker.patch("freqtrade.wallets.Wallets.get_total", return_value=123) + + create_mock_trades(fee, is_short=None) + trade = Trade.session.scalars(select(Trade)).first() + assert trade.amount == 123 + + assert freqtrade.wallets.check_exit_amount(trade) is True + assert update_mock.call_count == 0 + assert total_mock.call_count == 1 + + update_mock.reset_mock() + # Reduce returned amount to below the trade amount - which should + # trigger a wallet update and return False, triggering "order refinding" + total_mock = mocker.patch("freqtrade.wallets.Wallets.get_total", return_value=100) + assert freqtrade.wallets.check_exit_amount(trade) is False + assert update_mock.call_count == 1 + assert total_mock.call_count == 2 + + +def test_check_exit_amount_futures(mocker, default_conf, fee): + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + freqtrade = get_patched_freqtradebot(mocker, default_conf) + total_mock = mocker.patch("freqtrade.wallets.Wallets.get_total", return_value=123) + + create_mock_trades(fee, is_short=None) + trade = Trade.session.scalars(select(Trade)).first() + trade.trading_mode = 'futures' + assert trade.amount == 123 + + assert freqtrade.wallets.check_exit_amount(trade) is True + assert total_mock.call_count == 0 + + update_mock = mocker.patch("freqtrade.wallets.Wallets.update") + trade.amount = 150 + # Reduce returned amount to below the trade amount - which should + # trigger a wallet update and return False, triggering "order refinding" + assert freqtrade.wallets.check_exit_amount(trade) is False + assert total_mock.call_count == 0 + assert update_mock.call_count == 1 From 491d2cb024066f38b4d25df9f2e1d4ad554f5489 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Apr 2023 20:32:51 +0200 Subject: [PATCH 15/16] Explicit test for handle_onexchange_order --- tests/test_freqtradebot.py | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ea99061b8..8aa3f63d5 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5552,6 +5552,51 @@ def test_handle_insufficient_funds(mocker, default_conf_usdt, fee, is_short, cap assert log_has(f"Error updating {order['id']}.", caplog) +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize("is_short", [False, True]) +def test_handle_onexchange_order(mocker, default_conf_usdt, limit_order, is_short, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mock_uts = mocker.spy(freqtrade, 'update_trade_state') + + entry_order = limit_order[entry_side(is_short)] + exit_order = limit_order[exit_side(is_short)] + mock_fo = mocker.patch(f'{EXMS}.fetch_orders', return_value=[ + entry_order, + exit_order, + ]) + + order_id = entry_order['id'] + + trade = Trade( + open_order_id=order_id, + pair='ETH/USDT', + fee_open=0.001, + fee_close=0.001, + open_rate=entry_order['price'], + open_date=arrow.utcnow().datetime, + stake_amount=entry_order['cost'], + amount=entry_order['amount'], + exchange="binance", + is_short=is_short, + leverage=1, + ) + + trade.orders.append(Order.parse_from_ccxt_object( + entry_order, 'ADA/USDT', entry_side(is_short)) + ) + Trade.session.add(trade) + freqtrade.handle_onexchange_order(trade) + assert log_has_re(r"Found previously unknown order .*", caplog) + assert mock_uts.call_count == 1 + assert mock_fo.call_count == 1 + + trade = Trade.session.scalars(select(Trade)).first() + + assert len(trade.orders) == 2 + assert trade.is_open is False + assert trade.exit_reason == ExitType.SOLD_ON_EXCHANGE.value + + def test_get_valid_price(mocker, default_conf_usdt) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 395ac5f6dc4289a7a9cf60751759b547b15eec47 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Apr 2023 06:23:34 +0200 Subject: [PATCH 16/16] Update integration test --- tests/test_integration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 9fb9fd8b3..2949f1ef2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -75,8 +75,9 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, _notify_exit=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_exit", should_sell_mock) - wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) - mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1000)) + wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update") + mocker.patch("freqtrade.wallets.Wallets.get_free", return_value=1000) + mocker.patch("freqtrade.wallets.Wallets.check_exit_amount", return_value=True) freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True