From 54d467d2bf6876826985a2cf30a5ec53cb6ec8ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 10:30:27 +0200 Subject: [PATCH 1/8] Add Post endpoint for to filter dataframe by columns --- freqtrade/rpc/api_server/api_schemas.py | 8 +++++++ freqtrade/rpc/api_server/api_v1.py | 23 ++++++++++++------ freqtrade/rpc/rpc.py | 32 +++++++++++++++++-------- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 97f851b1d..cd60b3d92 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -489,12 +489,20 @@ class AvailablePairs(BaseModel): pair_interval: List[List[str]] +class PairHistoryRequest(BaseModel): + pair: str + timeframe: str + limit: Optional[int] = None + columns: Optional[List[str]] = None + + class PairHistory(BaseModel): strategy: str pair: str timeframe: str timeframe_ms: int columns: List[str] + selected_columns: List[str] = [] data: SerializeAsAny[List[Any]] length: int buy_signals: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8146fe276..c180012e2 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -17,10 +17,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, LocksPayload, Logs, MixTag, OpenTradeSchema, - PairHistory, PerformanceEntry, Ping, PlotConfig, - Profit, ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistResponse) + PairHistory, PairHistoryRequest, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, ShowConfig, + Stats, StatusMsg, StrategyListResponse, + StrategyResponse, SysInfo, Version, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -53,7 +54,8 @@ logger = logging.getLogger(__name__) # 2.32: new /backtest/history/ patch endpoint # 2.33: Additional weekly/monthly metrics # 2.34: new entries/exits/mix_tags endpoints -API_VERSION = 2.34 +# 2.35: pair_candles and pair_history endpoints as Post variant +API_VERSION = 2.35 # Public API, requires no auth. router_public = APIRouter() @@ -291,7 +293,14 @@ def reload_config(rpc: RPC = Depends(get_rpc)): @router.get('/pair_candles', response_model=PairHistory, tags=['candle data']) def pair_candles( pair: str, timeframe: str, limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_analysed_dataframe(pair, timeframe, limit) + return rpc._rpc_analysed_dataframe(pair, timeframe, limit, None) + + +@router.post('/pair_candles', response_model=PairHistory, tags=['candle data']) +def pair_candles_filtered(payload: PairHistoryRequest, rpc: RPC = Depends(get_rpc)): + # Advanced pair_candles endpoint with column filtering + return rpc._rpc_analysed_dataframe( + payload.pair, payload.timeframe, payload.limit, payload.columns) @router.get('/pair_history', response_model=PairHistory, tags=['candle data']) @@ -307,7 +316,7 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, 'freqaimodel': freqaimodel if freqaimodel else config.get('freqaimodel'), }) try: - return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange) + return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange, None) except Exception as e: raise HTTPException(status_code=502, detail=str(e)) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 43be0fd94..7ab9f9413 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,7 @@ from sqlalchemy import func, select from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange -from freqtrade.constants import CANCEL_REASON, Config +from freqtrade.constants import CANCEL_REASON, DEFAULT_DATAFRAME_COLUMNS, Config from freqtrade.data.history import load_data from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection, @@ -1190,9 +1190,11 @@ class RPC: return self._freqtrade.edge.accepted_pairs() @staticmethod - def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame, - last_analyzed: datetime) -> Dict[str, Any]: + def _convert_dataframe_to_dict( + strategy: str, pair: str, timeframe: str, dataframe: DataFrame, + last_analyzed: datetime, selected_cols: Optional[List[str]]) -> Dict[str, Any]: has_content = len(dataframe) != 0 + dataframe_columns = list(dataframe.columns) signals = { 'enter_long': 0, 'exit_long': 0, @@ -1200,6 +1202,11 @@ class RPC: 'exit_short': 0, } if has_content: + if selected_cols: + # Ensure OHLCV columns are always present + cols_set = set(DEFAULT_DATAFRAME_COLUMNS + selected_cols) + df_cols = [col for col in dataframe_columns if col in cols_set] + dataframe = dataframe.loc[:, df_cols] dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 # Move signal close to separate column when signal for easy plotting @@ -1224,7 +1231,8 @@ class RPC: 'timeframe': timeframe, 'timeframe_ms': timeframe_to_msecs(timeframe), 'strategy': strategy, - 'columns': list(dataframe.columns), + 'columns': dataframe_columns, + 'selected_columns': list(dataframe.columns), 'data': dataframe.values.tolist(), 'length': len(dataframe), 'buy_signals': signals['enter_long'], # Deprecated @@ -1249,13 +1257,16 @@ class RPC: }) return res - def _rpc_analysed_dataframe(self, pair: str, timeframe: str, - limit: Optional[int]) -> Dict[str, Any]: + def _rpc_analysed_dataframe( + self, pair: str, timeframe: str, limit: Optional[int], + selected_cols: Optional[List[str]]) -> Dict[str, Any]: """ Analyzed dataframe in Dict form """ _data, last_analyzed = self.__rpc_analysed_dataframe_raw(pair, timeframe, limit) - return RPC._convert_dataframe_to_dict(self._freqtrade.config['strategy'], - pair, timeframe, _data, last_analyzed) + return RPC._convert_dataframe_to_dict( + self._freqtrade.config['strategy'], pair, timeframe, _data, last_analyzed, + selected_cols + ) def __rpc_analysed_dataframe_raw( self, @@ -1322,7 +1333,7 @@ class RPC: @staticmethod def _rpc_analysed_history_full(config: Config, pair: str, timeframe: str, - exchange) -> Dict[str, Any]: + exchange, selected_cols: Optional[List[str]]) -> Dict[str, Any]: timerange_parsed = TimeRange.parse_timerange(config.get('timerange')) from freqtrade.data.converter import trim_dataframe @@ -1352,7 +1363,8 @@ class RPC: df_analyzed = trim_dataframe(df_analyzed, timerange_parsed, startup_candles=startup_candles) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, - df_analyzed.copy(), dt_now()) + df_analyzed.copy(), dt_now(), + selected_cols) def _rpc_plot_config(self) -> Dict[str, Any]: if (self._freqtrade.strategy.plot_config and From 3e2077044617ae5f40c1c42d9a2e2074c0aa7141 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 10:40:54 +0200 Subject: [PATCH 2/8] Slightly change returned column scheme --- freqtrade/rpc/api_server/api_schemas.py | 2 +- freqtrade/rpc/rpc.py | 6 ++-- tests/rpc/test_rpc_apiserver.py | 42 ++++++++++++++----------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index cd60b3d92..7b6d5a3f6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -502,7 +502,7 @@ class PairHistory(BaseModel): timeframe: str timeframe_ms: int columns: List[str] - selected_columns: List[str] = [] + all_columns: List[str] = [] data: SerializeAsAny[List[Any]] length: int buy_signals: int diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7ab9f9413..ed99b46ac 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1204,7 +1204,7 @@ class RPC: if has_content: if selected_cols: # Ensure OHLCV columns are always present - cols_set = set(DEFAULT_DATAFRAME_COLUMNS + selected_cols) + cols_set = set(DEFAULT_DATAFRAME_COLUMNS + list(signals.keys()) + selected_cols) df_cols = [col for col in dataframe_columns if col in cols_set] dataframe = dataframe.loc[:, df_cols] @@ -1231,8 +1231,8 @@ class RPC: 'timeframe': timeframe, 'timeframe_ms': timeframe_to_msecs(timeframe), 'strategy': strategy, - 'columns': dataframe_columns, - 'selected_columns': list(dataframe.columns), + 'all_columns': dataframe_columns, + 'columns': list(dataframe.columns), 'data': dataframe.values.tolist(), 'length': len(dataframe), 'buy_signals': signals['enter_long'], # Deprecated diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5b189371a..08144693c 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1522,31 +1522,37 @@ def test_api_pair_candles(botclient, ohlcv_history): rc = client_get(client, f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") assert_response(rc) - assert 'strategy' in rc.json() - assert rc.json()['strategy'] == CURRENT_TEST_STRATEGY - assert 'columns' in rc.json() - assert 'data_start_ts' in rc.json() - assert 'data_start' in rc.json() - assert 'data_stop' in rc.json() - assert 'data_stop_ts' in rc.json() - assert rc.json()['data_start'] == '2017-11-26 08:50:00+00:00' - assert rc.json()['data_start_ts'] == 1511686200000 - assert rc.json()['data_stop'] == '2017-11-26 09:00:00+00:00' - assert rc.json()['data_stop_ts'] == 1511686800000 - assert isinstance(rc.json()['columns'], list) - assert set(rc.json()['columns']) == { + resp = rc.json() + assert 'strategy' in resp + assert resp['strategy'] == CURRENT_TEST_STRATEGY + assert 'columns' in resp + assert 'data_start_ts' in resp + assert 'data_start' in resp + assert 'data_stop' in resp + assert 'data_stop_ts' in resp + assert resp['data_start'] == '2017-11-26 08:50:00+00:00' + assert resp['data_start_ts'] == 1511686200000 + assert resp['data_stop'] == '2017-11-26 09:00:00+00:00' + assert resp['data_stop_ts'] == 1511686800000 + assert isinstance(resp['columns'], list) + assert set(resp['columns']) == { 'date', 'open', 'high', 'low', 'close', 'volume', 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', '_enter_long_signal_close', '_exit_long_signal_close', '_enter_short_signal_close', '_exit_short_signal_close' } - assert 'pair' in rc.json() - assert rc.json()['pair'] == 'XRP/BTC' + # All columns doesn't include the internal columns + assert set(resp['all_columns']) == { + 'date', 'open', 'high', 'low', 'close', 'volume', + 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short' + } + assert 'pair' in resp + assert resp['pair'] == 'XRP/BTC' - assert 'data' in rc.json() - assert len(rc.json()['data']) == amount + assert 'data' in resp + assert len(resp['data']) == amount - assert (rc.json()['data'] == + assert (resp['data'] == [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, From eeaa9061e58cb4e2400fad574fc89113e9603523 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 10:49:50 +0200 Subject: [PATCH 3/8] Adapt test for to also test post endpoint --- tests/rpc/test_rpc_apiserver.py | 109 ++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 08144693c..66119b719 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1511,6 +1511,7 @@ def test_api_pair_candles(botclient, ohlcv_history): assert 'data_stop_ts' in rc.json() assert len(rc.json()['data']) == 0 ohlcv_history['sma'] = ohlcv_history['close'].rolling(2).mean() + ohlcv_history['sma2'] = ohlcv_history['close'].rolling(2).mean() ohlcv_history['enter_long'] = 0 ohlcv_history.loc[1, 'enter_long'] = 1 ohlcv_history['exit_long'] = 0 @@ -1518,50 +1519,66 @@ def test_api_pair_candles(botclient, ohlcv_history): ohlcv_history['exit_short'] = 0 ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT) + for call in ('get', 'post'): + if call == 'get': + rc = client_get( + client, + f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_candles", + data={ + "pair": "XRP/BTC", + "timeframe": timeframe, + "limit": amount, + "columns": ['sma'], + } + ) + assert_response(rc) + resp = rc.json() + assert 'strategy' in resp + assert resp['strategy'] == CURRENT_TEST_STRATEGY + assert 'columns' in resp + assert 'data_start_ts' in resp + assert 'data_start' in resp + assert 'data_stop' in resp + assert 'data_stop_ts' in resp + assert resp['data_start'] == '2017-11-26 08:50:00+00:00' + assert resp['data_start_ts'] == 1511686200000 + assert resp['data_stop'] == '2017-11-26 09:00:00+00:00' + assert resp['data_stop_ts'] == 1511686800000 + assert isinstance(resp['columns'], list) + if call == 'get': + assert set(resp['columns']) == { + 'date', 'open', 'high', 'low', 'close', 'volume', + 'sma', 'sma2', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', + '_enter_long_signal_close', '_exit_long_signal_close', + '_enter_short_signal_close', '_exit_short_signal_close' + } + # All columns doesn't include the internal columns + assert set(resp['all_columns']) == { + 'date', 'open', 'high', 'low', 'close', 'volume', + 'sma', 'sma2', 'enter_long', 'exit_long', 'enter_short', 'exit_short' + } + assert 'pair' in resp + assert resp['pair'] == 'XRP/BTC' - rc = client_get(client, - f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") - assert_response(rc) - resp = rc.json() - assert 'strategy' in resp - assert resp['strategy'] == CURRENT_TEST_STRATEGY - assert 'columns' in resp - assert 'data_start_ts' in resp - assert 'data_start' in resp - assert 'data_stop' in resp - assert 'data_stop_ts' in resp - assert resp['data_start'] == '2017-11-26 08:50:00+00:00' - assert resp['data_start_ts'] == 1511686200000 - assert resp['data_stop'] == '2017-11-26 09:00:00+00:00' - assert resp['data_stop_ts'] == 1511686800000 - assert isinstance(resp['columns'], list) - assert set(resp['columns']) == { - 'date', 'open', 'high', 'low', 'close', 'volume', - 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', - '_enter_long_signal_close', '_exit_long_signal_close', - '_enter_short_signal_close', '_exit_short_signal_close' - } - # All columns doesn't include the internal columns - assert set(resp['all_columns']) == { - 'date', 'open', 'high', 'low', 'close', 'volume', - 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short' - } - assert 'pair' in resp - assert resp['pair'] == 'XRP/BTC' + assert 'data' in resp + assert len(resp['data']) == amount + if call == 'get': + assert resp['data'] == [ + ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, + 0.0877869, None, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, + 8.886500000000001e-05, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, + 8.893e-05, None, None, None], + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None + ] + ] - assert 'data' in resp - assert len(resp['data']) == amount - - assert (resp['data'] == - [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, - None, 0, 0, 0, 0, 1511686200000, None, None, None, None], - ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, 8.893e-05, - None, None, None], - ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None] - - ]) + # prep for next test ohlcv_history['exit_long'] = ohlcv_history['exit_long'].astype('float64') ohlcv_history.at[0, 'exit_long'] = float('inf') ohlcv_history['date1'] = ohlcv_history['date'] @@ -1573,12 +1590,12 @@ def test_api_pair_candles(botclient, ohlcv_history): assert_response(rc) assert (rc.json()['data'] == [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, - None, 0, None, 0, 0, None, 1511686200000, None, None, None, None], + None, None, 0, None, 0, 0, None, 1511686200000, None, None, None, None], ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26T08:55:00Z', - 1511686500000, 8.893e-05, None, None, None], + 8.893e-05, 0.05874751, 8.886500000000001e-05, 8.886500000000001e-05, 1, 0.0, 0, + 0, '2017-11-26T08:55:00Z', 1511686500000, 8.893e-05, None, None, None], ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000, + 0.7039405, 8.885e-05, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000, None, None, None, None] ]) From 8c39740105b9986cd31b495b807c94a4dd4862e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 10:51:53 +0200 Subject: [PATCH 4/8] Test Post pair_candles endpoint --- tests/rpc/test_rpc_apiserver.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 66119b719..13d91d494 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1549,13 +1549,17 @@ def test_api_pair_candles(botclient, ohlcv_history): assert resp['data_stop'] == '2017-11-26 09:00:00+00:00' assert resp['data_stop_ts'] == 1511686800000 assert isinstance(resp['columns'], list) - if call == 'get': - assert set(resp['columns']) == { + base_cols = { 'date', 'open', 'high', 'low', 'close', 'volume', - 'sma', 'sma2', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', + 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', '_enter_long_signal_close', '_exit_long_signal_close', '_enter_short_signal_close', '_exit_short_signal_close' } + if call == 'get': + assert set(resp['columns']) == base_cols.union({'sma2'}) + else: + assert set(resp['columns']) == base_cols + # All columns doesn't include the internal columns assert set(resp['all_columns']) == { 'date', 'open', 'high', 'low', 'close', 'volume', @@ -1567,6 +1571,7 @@ def test_api_pair_candles(botclient, ohlcv_history): assert 'data' in resp assert len(resp['data']) == amount if call == 'get': + assert len(resp['data'][0]) == 17 assert resp['data'] == [ ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], @@ -1577,6 +1582,18 @@ def test_api_pair_candles(botclient, ohlcv_history): 0.7039405, 8.885e-05, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None ] ] + else: + assert len(resp['data'][0]) == 16 + assert resp['data'] == [ + ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, + 0.0877869, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, + 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, + 8.893e-05, None, None, None], + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None + ] + ] # prep for next test ohlcv_history['exit_long'] = ohlcv_history['exit_long'].astype('float64') @@ -1595,8 +1612,8 @@ def test_api_pair_candles(botclient, ohlcv_history): 8.893e-05, 0.05874751, 8.886500000000001e-05, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26T08:55:00Z', 1511686500000, 8.893e-05, None, None, None], ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000, - None, None, None, None] + 0.7039405, 8.885e-05, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', + 1511686800000, None, None, None, None] ]) From ccd788e2ce8719c48825982fdc0cca7af5cad2a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 11:33:45 +0200 Subject: [PATCH 5/8] Improve naming of the schema --- freqtrade/rpc/api_server/api_schemas.py | 2 +- freqtrade/rpc/api_server/api_v1.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 7b6d5a3f6..1b4d46e79 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -489,7 +489,7 @@ class AvailablePairs(BaseModel): pair_interval: List[List[str]] -class PairHistoryRequest(BaseModel): +class PairCandlesRequest(BaseModel): pair: str timeframe: str limit: Optional[int] = None diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index c180012e2..d4c2d80cb 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -17,7 +17,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, LocksPayload, Logs, MixTag, OpenTradeSchema, - PairHistory, PairHistoryRequest, PerformanceEntry, + PairCandlesRequest, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, SysInfo, Version, @@ -297,7 +297,7 @@ def pair_candles( @router.post('/pair_candles', response_model=PairHistory, tags=['candle data']) -def pair_candles_filtered(payload: PairHistoryRequest, rpc: RPC = Depends(get_rpc)): +def pair_candles_filtered(payload: PairCandlesRequest, rpc: RPC = Depends(get_rpc)): # Advanced pair_candles endpoint with column filtering return rpc._rpc_analysed_dataframe( payload.pair, payload.timeframe, payload.limit, payload.columns) From ab103798332360b25851470c8222011870379bed Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 11:44:21 +0200 Subject: [PATCH 6/8] Add pair_history post endpoint, too --- freqtrade/rpc/api_server/api_schemas.py | 6 +++ freqtrade/rpc/api_server/api_v1.py | 28 ++++++++-- tests/rpc/test_rpc_apiserver.py | 69 +++++++++++++++---------- 3 files changed, 72 insertions(+), 31 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 1b4d46e79..4e20a0f9d 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -496,6 +496,12 @@ class PairCandlesRequest(BaseModel): columns: Optional[List[str]] = None +class PairHistoryRequest(PairCandlesRequest): + timerange: str + strategy: str + freqaimodel: Optional[str] = None + + class PairHistory(BaseModel): strategy: str pair: str diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d4c2d80cb..2d3ba32fc 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -17,11 +17,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, LocksPayload, Logs, MixTag, OpenTradeSchema, - PairCandlesRequest, PairHistory, PerformanceEntry, - Ping, PlotConfig, Profit, ResultMsg, ShowConfig, - Stats, StatusMsg, StrategyListResponse, - StrategyResponse, SysInfo, Version, - WhitelistResponse) + PairCandlesRequest, PairHistory, + PairHistoryRequest, PerformanceEntry, Ping, + PlotConfig, Profit, ResultMsg, ShowConfig, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -321,6 +321,24 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, raise HTTPException(status_code=502, detail=str(e)) +@router.post('/pair_history', response_model=PairHistory, tags=['candle data']) +def pair_history_filtered(payload: PairHistoryRequest, + config=Depends(get_config), exchange=Depends(get_exchange)): + # The initial call to this endpoint can be slow, as it may need to initialize + # the exchange class. + config = deepcopy(config) + config.update({ + 'strategy': payload.strategy, + 'timerange': payload.timerange, + 'freqaimodel': payload.freqaimodel if payload.freqaimodel else config.get('freqaimodel'), + }) + try: + return RPC._rpc_analysed_history_full( + config, payload.pair, payload.timeframe, exchange, payload.columns) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)) + + @router.get('/plot_config', response_model=PlotConfig, tags=['candle data']) def plot_config(strategy: Optional[str] = None, config=Depends(get_config), rpc: Optional[RPC] = Depends(get_rpc_optional)): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 13d91d494..6fae4aaee 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1654,33 +1654,50 @@ def test_api_pair_history(botclient, tmp_path, mocker): assert_response(rc, 502) # Working - rc = client_get(client, - f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" - f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}") - assert_response(rc, 200) - result = rc.json() - assert result['length'] == 289 - assert len(result['data']) == result['length'] - assert 'columns' in result - assert 'data' in result - data = result['data'] - assert len(data) == 289 - # analyzed DF has 30 columns - assert len(result['columns']) == 30 - assert len(data[0]) == 30 - date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] - rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] + for call in ('get', 'post'): + if call == 'get': + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_history", + data={ + "pair": "UNITTEST/BTC", + "timeframe": timeframe, + "timerange": "20180111-20180112", + "strategy": CURRENT_TEST_STRATEGY, + "columns": ['rsi', 'fastd', 'fastk'], + }) - assert data[0][date_col_idx] == '2018-01-11T00:00:00Z' - assert data[0][rsi_col_idx] is not None - assert data[0][rsi_col_idx] > 0 - assert lfm.call_count == 1 - assert result['pair'] == 'UNITTEST/BTC' - assert result['strategy'] == CURRENT_TEST_STRATEGY - assert result['data_start'] == '2018-01-11 00:00:00+00:00' - assert result['data_start_ts'] == 1515628800000 - assert result['data_stop'] == '2018-01-12 00:00:00+00:00' - assert result['data_stop_ts'] == 1515715200000 + assert_response(rc, 200) + result = rc.json() + assert result['length'] == 289 + assert len(result['data']) == result['length'] + assert 'columns' in result + assert 'data' in result + data = result['data'] + assert len(data) == 289 + col_count = 30 if call == 'get' else 18 + # analyzed DF has 30 columns + assert len(result['columns']) == col_count + assert len(result['all_columns']) == 25 + assert len(data[0]) == col_count + date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] + rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] + + assert data[0][date_col_idx] == '2018-01-11T00:00:00Z' + assert data[0][rsi_col_idx] is not None + assert data[0][rsi_col_idx] > 0 + assert lfm.call_count == 1 + assert result['pair'] == 'UNITTEST/BTC' + assert result['strategy'] == CURRENT_TEST_STRATEGY + assert result['data_start'] == '2018-01-11 00:00:00+00:00' + assert result['data_start_ts'] == 1515628800000 + assert result['data_stop'] == '2018-01-12 00:00:00+00:00' + assert result['data_stop_ts'] == 1515715200000 + lfm.reset_mock() # No data found rc = client_get(client, From 9d57e3930da0474a6e30c1fb77aa8133734565a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 17:01:16 +0200 Subject: [PATCH 7/8] Extend error testing to post call --- tests/rpc/test_rpc_apiserver.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6fae4aaee..c88e18d05 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1699,12 +1699,24 @@ def test_api_pair_history(botclient, tmp_path, mocker): assert result['data_stop_ts'] == 1515715200000 lfm.reset_mock() - # No data found - rc = client_get(client, - f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" - f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}") - assert_response(rc, 502) - assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.") + # No data found + if call == 'get': + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_history", + data={ + "pair": "UNITTEST/BTC", + "timeframe": timeframe, + "timerange": "20200111-20200112", + "strategy": CURRENT_TEST_STRATEGY, + "columns": ['rsi', 'fastd', 'fastk'], + }) + assert_response(rc, 502) + assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.") def test_api_plot_config(botclient, mocker, tmp_path): From 1649aca411ff833405f0035383a0df8dfa87d802 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Apr 2024 18:22:37 +0200 Subject: [PATCH 8/8] Support empty array to return only the base columns --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ed99b46ac..87f5688c7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1202,7 +1202,7 @@ class RPC: 'exit_short': 0, } if has_content: - if selected_cols: + if selected_cols is not None: # Ensure OHLCV columns are always present cols_set = set(DEFAULT_DATAFRAME_COLUMNS + list(signals.keys()) + selected_cols) df_cols = [col for col in dataframe_columns if col in cols_set]