From 48e5a458561a8aa8e9ade9f4c9863d0d9e659c45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 10:54:03 +0000 Subject: [PATCH 1/6] rpc_test: dont replicate whole response, updating what's changed improves readability --- tests/rpc/test_rpc.py | 257 ++++++++++++------------------------------ 1 file changed, 74 insertions(+), 183 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index c3a0d539d..8725ad2c5 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 # pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments +from copy import deepcopy from datetime import datetime, timedelta, timezone from unittest.mock import ANY, MagicMock, PropertyMock @@ -28,113 +29,7 @@ def prec_satoshi(a, b) -> float: # Unit tests def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: - mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - fetch_ticker=ticker, - get_fee=fee, - _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot) - rpc = RPC(freqtradebot) - - freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*no active trade*'): - rpc._rpc_trade_status() - - freqtradebot.enter_positions() - - # Open order... - results = rpc._rpc_trade_status() - assert results[0] == { - 'trade_id': 1, - 'pair': 'ETH/BTC', - 'base_currency': 'ETH', - 'quote_currency': 'BTC', - 'open_date': ANY, - 'open_timestamp': ANY, - 'is_open': ANY, - 'fee_open': ANY, - 'fee_open_cost': ANY, - 'fee_open_currency': ANY, - 'fee_close': fee.return_value, - 'fee_close_cost': ANY, - 'fee_close_currency': ANY, - 'open_rate_requested': ANY, - 'open_trade_value': 0.0010025, - 'close_rate_requested': ANY, - 'sell_reason': ANY, - 'exit_reason': ANY, - 'exit_order_status': ANY, - 'min_rate': ANY, - 'max_rate': ANY, - 'strategy': ANY, - 'buy_tag': ANY, - 'enter_tag': ANY, - 'timeframe': 5, - 'open_order_id': ANY, - 'close_date': None, - 'close_timestamp': None, - 'open_rate': 1.098e-05, - 'close_rate': None, - 'current_rate': 1.099e-05, - 'amount': 91.07468124, - 'amount_requested': 91.07468124, - 'stake_amount': 0.001, - 'trade_duration': None, - 'trade_duration_s': None, - 'close_profit': None, - 'close_profit_pct': None, - 'close_profit_abs': None, - 'current_profit': 0.0, - 'current_profit_pct': 0.0, - 'current_profit_abs': 0.0, - 'profit_ratio': 0.0, - 'profit_pct': 0.0, - 'profit_abs': 0.0, - 'profit_fiat': ANY, - 'stop_loss_abs': 0.0, - 'stop_loss_pct': None, - 'stop_loss_ratio': None, - 'stoploss_order_id': None, - 'stoploss_last_update': ANY, - 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 0.0, - 'initial_stop_loss_pct': None, - 'initial_stop_loss_ratio': None, - 'stoploss_current_dist': -1.099e-05, - 'stoploss_current_dist_ratio': -1.0, - 'stoploss_current_dist_pct': pytest.approx(-100.0), - 'stoploss_entry_dist': -0.0010025, - 'stoploss_entry_dist_ratio': -1.0, - 'open_order': '(limit buy rem=91.07468123)', - 'realized_profit': 0.0, - 'exchange': 'binance', - 'leverage': 1.0, - 'interest_rate': 0.0, - 'liquidation_price': None, - 'is_short': False, - 'funding_fees': 0.0, - 'trading_mode': TradingMode.SPOT, - 'orders': [{ - 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, - 'cost': 0.0009999999999054, 'filled': 0.0, 'ft_order_side': 'buy', - 'order_date': ANY, 'order_timestamp': ANY, 'order_filled_date': ANY, - 'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05, - 'is_open': True, 'pair': 'ETH/BTC', 'order_id': ANY, - 'remaining': 91.07468123, 'status': ANY, 'ft_is_entry': True, - }], - } - - # Fill open order ... - freqtradebot.manage_open_orders() - trades = Trade.get_open_trades() - freqtradebot.exit_positions(trades) - - results = rpc._rpc_trade_status() - assert results[0] == { + gen_response = { 'trade_id': 1, 'pair': 'ETH/BTC', 'base_currency': 'ETH', @@ -213,91 +108,87 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'remaining': ANY, 'status': ANY, 'ft_is_entry': True, }], } + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), + ) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot) + rpc = RPC(freqtradebot) + + freqtradebot.state = State.RUNNING + with pytest.raises(RPCException, match=r'.*no active trade*'): + rpc._rpc_trade_status() + + freqtradebot.enter_positions() + + # Open order... + results = rpc._rpc_trade_status() + response_unfilled = deepcopy(gen_response) + # Different from "filled" response: + response_unfilled.update({ + 'amount': 91.07468124, + 'profit_ratio': 0.0, + 'profit_pct': 0.0, + 'profit_abs': 0.0, + 'current_profit': 0.0, + 'current_profit_pct': 0.0, + 'current_profit_abs': 0.0, + 'stop_loss_abs': 0.0, + 'stop_loss_pct': None, + 'stop_loss_ratio': None, + 'stoploss_current_dist': -1.099e-05, + 'stoploss_current_dist_ratio': -1.0, + 'stoploss_current_dist_pct': pytest.approx(-100.0), + 'stoploss_entry_dist': -0.0010025, + 'stoploss_entry_dist_ratio': -1.0, + 'initial_stop_loss_abs': 0.0, + 'initial_stop_loss_pct': None, + 'initial_stop_loss_ratio': None, + 'open_order': '(limit buy rem=91.07468123)', + }) + response_unfilled['orders'][0].update({ + 'is_open': True, + 'filled': 0.0, + 'remaining': 91.07468123 + }) + + assert results[0] == response_unfilled + + # Fill open order ... + freqtradebot.manage_open_orders() + trades = Trade.get_open_trades() + freqtradebot.exit_positions(trades) + + results = rpc._rpc_trade_status() + + response = deepcopy(gen_response) + assert results[0] == response mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) - assert results[0] == { - 'trade_id': 1, - 'pair': 'ETH/BTC', - 'base_currency': 'ETH', - 'quote_currency': 'BTC', - 'open_date': ANY, - 'open_timestamp': ANY, - 'is_open': ANY, - 'fee_open': ANY, - 'fee_open_cost': ANY, - 'fee_open_currency': ANY, - 'fee_close': fee.return_value, - 'fee_close_cost': ANY, - 'fee_close_currency': ANY, - 'open_rate_requested': ANY, - 'open_trade_value': ANY, - 'close_rate_requested': ANY, - 'sell_reason': ANY, - 'exit_reason': ANY, - 'exit_order_status': ANY, - 'min_rate': ANY, - 'max_rate': ANY, - 'strategy': ANY, - 'buy_tag': ANY, - 'enter_tag': ANY, - 'timeframe': ANY, - 'open_order_id': ANY, - 'close_date': None, - 'close_timestamp': None, - 'open_rate': 1.098e-05, - 'close_rate': None, - 'current_rate': ANY, - 'amount': 91.07468123, - 'amount_requested': 91.07468124, - 'trade_duration': ANY, - 'trade_duration_s': ANY, - 'stake_amount': 0.001, - 'close_profit': None, - 'close_profit_pct': None, - 'close_profit_abs': None, - 'current_profit': ANY, - 'current_profit_pct': ANY, - 'current_profit_abs': ANY, - 'profit_ratio': ANY, - 'profit_pct': ANY, - 'profit_abs': ANY, - 'profit_fiat': ANY, - 'stop_loss_abs': 9.89e-06, - 'stop_loss_pct': -10.0, - 'stop_loss_ratio': -0.1, - 'stoploss_order_id': None, - 'stoploss_last_update': ANY, - 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 9.89e-06, - 'initial_stop_loss_pct': -10.0, - 'initial_stop_loss_ratio': -0.1, + response_norate = deepcopy(gen_response) + # Update elements that are NaN when no rate is available. + response_norate.update({ 'stoploss_current_dist': ANY, 'stoploss_current_dist_ratio': ANY, 'stoploss_current_dist_pct': ANY, - 'stoploss_entry_dist': -0.00010402, - 'stoploss_entry_dist_ratio': -0.10376381, - 'open_order': None, - 'exchange': 'binance', - 'realized_profit': 0.0, - 'leverage': 1.0, - 'interest_rate': 0.0, - 'liquidation_price': None, - 'is_short': False, - 'funding_fees': 0.0, - 'trading_mode': TradingMode.SPOT, - 'orders': [{ - 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, - 'cost': 0.0009999999999054, 'filled': 91.07468123, 'ft_order_side': 'buy', - 'order_date': ANY, 'order_timestamp': ANY, 'order_filled_date': ANY, - 'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05, - 'is_open': False, 'pair': 'ETH/BTC', 'order_id': ANY, - 'remaining': ANY, 'status': ANY, 'ft_is_entry': True, - }], - } + 'profit_ratio': ANY, + 'profit_pct': ANY, + 'profit_abs': ANY, + 'current_profit_abs': ANY, + 'current_profit': ANY, + 'current_profit_pct': ANY, + 'current_rate': ANY, + }) + assert results[0] == response_norate def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: From 1975e942d68177286f54af33904da1c86290e6af Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 1 Nov 2022 18:58:04 +0000 Subject: [PATCH 2/6] Add test for no remaining (kucoin case - https://github.com/freqtrade/freqtrade/issues/7757). --- tests/rpc/test_rpc.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 8725ad2c5..ef6c8b204 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -156,9 +156,25 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'filled': 0.0, 'remaining': 91.07468123 }) - assert results[0] == response_unfilled + # Open order without remaining + trade = Trade.get_open_trades()[0] + # kucoin case (no remaining set). + trade.orders[0].remaining = None + Trade.commit() + + results = rpc._rpc_trade_status() + # Reuse above object, only remaining changed. + response_unfilled['orders'][0].update({ + 'remaining': None + }) + assert results[0] == response_unfilled + + trade = Trade.get_open_trades()[0] + trade.orders[0].remaining = trade.amount + Trade.commit() + # Fill open order ... freqtradebot.manage_open_orders() trades = Trade.get_open_trades() From 436b314c80d2dd1ac8a38a6806d71213f5540d18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 18:58:46 +0000 Subject: [PATCH 3/6] add safe_remaining fixes #7757 --- freqtrade/persistence/trade_model.py | 7 +++++++ freqtrade/rpc/rpc.py | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index e032b6b96..19ba48fcd 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -90,6 +90,13 @@ class Order(_DECL_BASE): def safe_filled(self) -> float: return self.filled if self.filled is not None else self.amount or 0.0 + @property + def safe_remaining(self) -> float: + return ( + self.remaining if self.remaining is not None else + self.amount - (self.filled or 0.0) + ) + @property def safe_fee_base(self) -> float: return self.ft_fee_base or 0.0 diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 55bb5a34b..a0824bcc1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -218,9 +218,10 @@ class RPC: stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2), stoploss_entry_dist=stoploss_entry_dist, stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8), - open_order='({} {} rem={:.8f})'.format( - order.order_type, order.side, order.remaining - ) if order else None, + open_order=( + f'({order.order_type} {order.side} rem={order.safe_remaining:.8f})' if + order else None + ), )) results.append(trade_dict) return results From 4c7bb79c86d6167659e8dca643f7bf9cd55519af Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Nov 2022 13:59:29 +0100 Subject: [PATCH 4/6] Restore prior data transfer format --- freqtrade/misc.py | 20 +++++++++++++------- scripts/ws_client.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 308f0b32d..2d2c7513a 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union from typing.io import IO from urllib.parse import urlparse -import pandas +import orjson +import pandas as pd import rapidjson from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN @@ -256,7 +257,7 @@ def parse_db_uri_for_logging(uri: str): return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') -def dataframe_to_json(dataframe: pandas.DataFrame) -> str: +def dataframe_to_json(dataframe: pd.DataFrame) -> str: """ Serialize a DataFrame for transmission over the wire using JSON :param dataframe: A pandas DataFrame @@ -265,23 +266,28 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str: # https://github.com/pandas-dev/pandas/issues/24889 # https://github.com/pandas-dev/pandas/issues/40443 # We need to convert to a dict to avoid mem leak - return dataframe.to_dict(orient='tight') + def default(z): + if isinstance(z, pd.Timestamp): + return z.timestamp() * 1e3 + raise TypeError + + return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') -def json_to_dataframe(data: str) -> pandas.DataFrame: +def json_to_dataframe(data: str) -> pd.DataFrame: """ Deserialize JSON into a DataFrame :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.DataFrame.from_dict(data, orient='tight') + dataframe = pd.read_json(data, orient='split') if 'date' in dataframe.columns: - dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) + dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True) return dataframe -def remove_entry_exit_signals(dataframe: pandas.DataFrame): +def remove_entry_exit_signals(dataframe: pd.DataFrame): """ Remove Entry and Exit signals from a DataFrame diff --git a/scripts/ws_client.py b/scripts/ws_client.py index ff437e63e..090039cde 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -101,7 +101,7 @@ def json_deserialize(message): :param message: The message to deserialize """ def json_to_dataframe(data: str) -> pandas.DataFrame: - dataframe = pandas.DataFrame.from_dict(data, orient='tight') + dataframe = pandas.read_json(data, orient='split') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) From 12cd83453c18b9962ba80f6a6a13434748917456 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Nov 2022 14:03:56 +0100 Subject: [PATCH 5/6] Add warning when queue websocket queue becomes too full --- freqtrade/rpc/api_server/webserver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 6464ae44e..ec4907e67 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -194,6 +194,9 @@ class ApiServer(RPCHandler): try: while True: logger.debug("Getting queue messages...") + if (qsize := async_queue.qsize()) > 20: + # If the queue becomes too big for too long, this may indicate a problem. + logger.warning(f"Queue size now {qsize}") # Get data from queue message: WSMessageSchemaType = await async_queue.get() logger.debug(f"Found message of type: {message.get('type')}") From b6a8e421f1b2ddb7112773c686867cc6debacf88 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Fri, 18 Nov 2022 09:43:39 -0700 Subject: [PATCH 6/6] remove redundant timestamp conversion in ws serializer --- freqtrade/rpc/api_server/ws/serializer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py index 8d06746f7..6c402a100 100644 --- a/freqtrade/rpc/api_server/ws/serializer.py +++ b/freqtrade/rpc/api_server/ws/serializer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import orjson import rapidjson -from pandas import DataFrame, Timestamp +from pandas import DataFrame from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy @@ -52,11 +52,6 @@ def _json_default(z): '__type__': 'dataframe', '__value__': dataframe_to_json(z) } - # Pandas returns a Timestamp object, we need to - # convert it to a timestamp int (with ms) for orjson - # to handle it - if isinstance(z, Timestamp): - return z.timestamp() * 1e3 raise TypeError