diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 97f851b1d..4e20a0f9d 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -489,12 +489,26 @@ class AvailablePairs(BaseModel): pair_interval: List[List[str]] +class PairCandlesRequest(BaseModel): + pair: str + timeframe: str + limit: Optional[int] = None + columns: Optional[List[str]] = None + + +class PairHistoryRequest(PairCandlesRequest): + timerange: str + strategy: str + freqaimodel: Optional[str] = None + + class PairHistory(BaseModel): strategy: str pair: str timeframe: str timeframe_ms: int columns: List[str] + all_columns: List[str] = [] data: SerializeAsAny[List[Any]] length: int buy_signals: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8146fe276..2d3ba32fc 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -17,10 +17,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, LocksPayload, Logs, MixTag, OpenTradeSchema, - PairHistory, PerformanceEntry, Ping, PlotConfig, - Profit, ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistResponse) + PairCandlesRequest, PairHistory, + PairHistoryRequest, PerformanceEntry, Ping, + PlotConfig, Profit, ResultMsg, ShowConfig, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -53,7 +54,8 @@ logger = logging.getLogger(__name__) # 2.32: new /backtest/history/ patch endpoint # 2.33: Additional weekly/monthly metrics # 2.34: new entries/exits/mix_tags endpoints -API_VERSION = 2.34 +# 2.35: pair_candles and pair_history endpoints as Post variant +API_VERSION = 2.35 # Public API, requires no auth. router_public = APIRouter() @@ -291,7 +293,14 @@ def reload_config(rpc: RPC = Depends(get_rpc)): @router.get('/pair_candles', response_model=PairHistory, tags=['candle data']) def pair_candles( pair: str, timeframe: str, limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_analysed_dataframe(pair, timeframe, limit) + return rpc._rpc_analysed_dataframe(pair, timeframe, limit, None) + + +@router.post('/pair_candles', response_model=PairHistory, tags=['candle data']) +def pair_candles_filtered(payload: PairCandlesRequest, rpc: RPC = Depends(get_rpc)): + # Advanced pair_candles endpoint with column filtering + return rpc._rpc_analysed_dataframe( + payload.pair, payload.timeframe, payload.limit, payload.columns) @router.get('/pair_history', response_model=PairHistory, tags=['candle data']) @@ -307,7 +316,25 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, 'freqaimodel': freqaimodel if freqaimodel else config.get('freqaimodel'), }) try: - return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange) + return RPC._rpc_analysed_history_full(config, pair, timeframe, exchange, None) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)) + + +@router.post('/pair_history', response_model=PairHistory, tags=['candle data']) +def pair_history_filtered(payload: PairHistoryRequest, + config=Depends(get_config), exchange=Depends(get_exchange)): + # The initial call to this endpoint can be slow, as it may need to initialize + # the exchange class. + config = deepcopy(config) + config.update({ + 'strategy': payload.strategy, + 'timerange': payload.timerange, + 'freqaimodel': payload.freqaimodel if payload.freqaimodel else config.get('freqaimodel'), + }) + try: + return RPC._rpc_analysed_history_full( + config, payload.pair, payload.timeframe, exchange, payload.columns) except Exception as e: raise HTTPException(status_code=502, detail=str(e)) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 43be0fd94..87f5688c7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,7 @@ from sqlalchemy import func, select from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange -from freqtrade.constants import CANCEL_REASON, Config +from freqtrade.constants import CANCEL_REASON, DEFAULT_DATAFRAME_COLUMNS, Config from freqtrade.data.history import load_data from freqtrade.data.metrics import calculate_expectancy, calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, MarketDirection, SignalDirection, @@ -1190,9 +1190,11 @@ class RPC: return self._freqtrade.edge.accepted_pairs() @staticmethod - def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame, - last_analyzed: datetime) -> Dict[str, Any]: + def _convert_dataframe_to_dict( + strategy: str, pair: str, timeframe: str, dataframe: DataFrame, + last_analyzed: datetime, selected_cols: Optional[List[str]]) -> Dict[str, Any]: has_content = len(dataframe) != 0 + dataframe_columns = list(dataframe.columns) signals = { 'enter_long': 0, 'exit_long': 0, @@ -1200,6 +1202,11 @@ class RPC: 'exit_short': 0, } if has_content: + if selected_cols is not None: + # Ensure OHLCV columns are always present + cols_set = set(DEFAULT_DATAFRAME_COLUMNS + list(signals.keys()) + selected_cols) + df_cols = [col for col in dataframe_columns if col in cols_set] + dataframe = dataframe.loc[:, df_cols] dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 # Move signal close to separate column when signal for easy plotting @@ -1224,6 +1231,7 @@ class RPC: 'timeframe': timeframe, 'timeframe_ms': timeframe_to_msecs(timeframe), 'strategy': strategy, + 'all_columns': dataframe_columns, 'columns': list(dataframe.columns), 'data': dataframe.values.tolist(), 'length': len(dataframe), @@ -1249,13 +1257,16 @@ class RPC: }) return res - def _rpc_analysed_dataframe(self, pair: str, timeframe: str, - limit: Optional[int]) -> Dict[str, Any]: + def _rpc_analysed_dataframe( + self, pair: str, timeframe: str, limit: Optional[int], + selected_cols: Optional[List[str]]) -> Dict[str, Any]: """ Analyzed dataframe in Dict form """ _data, last_analyzed = self.__rpc_analysed_dataframe_raw(pair, timeframe, limit) - return RPC._convert_dataframe_to_dict(self._freqtrade.config['strategy'], - pair, timeframe, _data, last_analyzed) + return RPC._convert_dataframe_to_dict( + self._freqtrade.config['strategy'], pair, timeframe, _data, last_analyzed, + selected_cols + ) def __rpc_analysed_dataframe_raw( self, @@ -1322,7 +1333,7 @@ class RPC: @staticmethod def _rpc_analysed_history_full(config: Config, pair: str, timeframe: str, - exchange) -> Dict[str, Any]: + exchange, selected_cols: Optional[List[str]]) -> Dict[str, Any]: timerange_parsed = TimeRange.parse_timerange(config.get('timerange')) from freqtrade.data.converter import trim_dataframe @@ -1352,7 +1363,8 @@ class RPC: df_analyzed = trim_dataframe(df_analyzed, timerange_parsed, startup_candles=startup_candles) return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, - df_analyzed.copy(), dt_now()) + df_analyzed.copy(), dt_now(), + selected_cols) def _rpc_plot_config(self) -> Dict[str, Any]: if (self._freqtrade.strategy.plot_config and diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f1c81b49c..fd45cc84a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1511,6 +1511,7 @@ def test_api_pair_candles(botclient, ohlcv_history): assert 'data_stop_ts' in rc.json() assert len(rc.json()['data']) == 0 ohlcv_history['sma'] = ohlcv_history['close'].rolling(2).mean() + ohlcv_history['sma2'] = ohlcv_history['close'].rolling(2).mean() ohlcv_history['enter_long'] = 0 ohlcv_history.loc[1, 'enter_long'] = 1 ohlcv_history['exit_long'] = 0 @@ -1518,44 +1519,83 @@ def test_api_pair_candles(botclient, ohlcv_history): ohlcv_history['exit_short'] = 0 ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT) + for call in ('get', 'post'): + if call == 'get': + rc = client_get( + client, + f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_candles", + data={ + "pair": "XRP/BTC", + "timeframe": timeframe, + "limit": amount, + "columns": ['sma'], + } + ) + assert_response(rc) + resp = rc.json() + assert 'strategy' in resp + assert resp['strategy'] == CURRENT_TEST_STRATEGY + assert 'columns' in resp + assert 'data_start_ts' in resp + assert 'data_start' in resp + assert 'data_stop' in resp + assert 'data_stop_ts' in resp + assert resp['data_start'] == '2017-11-26 08:50:00+00:00' + assert resp['data_start_ts'] == 1511686200000 + assert resp['data_stop'] == '2017-11-26 09:00:00+00:00' + assert resp['data_stop_ts'] == 1511686800000 + assert isinstance(resp['columns'], list) + base_cols = { + 'date', 'open', 'high', 'low', 'close', 'volume', + 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', + '_enter_long_signal_close', '_exit_long_signal_close', + '_enter_short_signal_close', '_exit_short_signal_close' + } + if call == 'get': + assert set(resp['columns']) == base_cols.union({'sma2'}) + else: + assert set(resp['columns']) == base_cols - rc = client_get(client, - f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") - assert_response(rc) - assert 'strategy' in rc.json() - assert rc.json()['strategy'] == CURRENT_TEST_STRATEGY - assert 'columns' in rc.json() - assert 'data_start_ts' in rc.json() - assert 'data_start' in rc.json() - assert 'data_stop' in rc.json() - assert 'data_stop_ts' in rc.json() - assert rc.json()['data_start'] == '2017-11-26 08:50:00+00:00' - assert rc.json()['data_start_ts'] == 1511686200000 - assert rc.json()['data_stop'] == '2017-11-26 09:00:00+00:00' - assert rc.json()['data_stop_ts'] == 1511686800000 - assert isinstance(rc.json()['columns'], list) - assert set(rc.json()['columns']) == { - 'date', 'open', 'high', 'low', 'close', 'volume', - 'sma', 'enter_long', 'exit_long', 'enter_short', 'exit_short', '__date_ts', - '_enter_long_signal_close', '_exit_long_signal_close', - '_enter_short_signal_close', '_exit_short_signal_close' - } - assert 'pair' in rc.json() - assert rc.json()['pair'] == 'XRP/BTC' + # All columns doesn't include the internal columns + assert set(resp['all_columns']) == { + 'date', 'open', 'high', 'low', 'close', 'volume', + 'sma', 'sma2', 'enter_long', 'exit_long', 'enter_short', 'exit_short' + } + assert 'pair' in resp + assert resp['pair'] == 'XRP/BTC' - assert 'data' in rc.json() - assert len(rc.json()['data']) == amount + assert 'data' in resp + assert len(resp['data']) == amount + if call == 'get': + assert len(resp['data'][0]) == 17 + assert resp['data'] == [ + ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, + 0.0877869, None, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, + 8.886500000000001e-05, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, + 8.893e-05, None, None, None], + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None + ] + ] + else: + assert len(resp['data'][0]) == 16 + assert resp['data'] == [ + ['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, + 0.0877869, None, 0, 0, 0, 0, 1511686200000, None, None, None, None], + ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, + 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, + 8.893e-05, None, None, None], + ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None + ] + ] - assert (rc.json()['data'] == - [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, - None, 0, 0, 0, 0, 1511686200000, None, None, None, None], - ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 0, 0, 1511686500000, 8.893e-05, - None, None, None], - ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 0, 0, 0, 0, 1511686800000, None, None, None, None] - - ]) + # prep for next test ohlcv_history['exit_long'] = ohlcv_history['exit_long'].astype('float64') ohlcv_history.at[0, 'exit_long'] = float('inf') ohlcv_history['date1'] = ohlcv_history['date'] @@ -1567,13 +1607,13 @@ def test_api_pair_candles(botclient, ohlcv_history): assert_response(rc) assert (rc.json()['data'] == [['2017-11-26T08:50:00Z', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, - None, 0, None, 0, 0, None, 1511686200000, None, None, None, None], + None, None, 0, None, 0, 0, None, 1511686200000, None, None, None, None], ['2017-11-26T08:55:00Z', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, 0, 0, '2017-11-26T08:55:00Z', - 1511686500000, 8.893e-05, None, None, None], + 8.893e-05, 0.05874751, 8.886500000000001e-05, 8.886500000000001e-05, 1, 0.0, 0, + 0, '2017-11-26T08:55:00Z', 1511686500000, 8.893e-05, None, None, None], ['2017-11-26T09:00:00Z', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', 1511686800000, - None, None, None, None] + 0.7039405, 8.885e-05, 8.885e-05, 0, 0.0, 0, 0, '2017-11-26T09:00:00Z', + 1511686800000, None, None, None, None] ]) @@ -1614,40 +1654,69 @@ def test_api_pair_history(botclient, tmp_path, mocker): assert_response(rc, 502) # Working - rc = client_get(client, - f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" - f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}") - assert_response(rc, 200) - result = rc.json() - assert result['length'] == 289 - assert len(result['data']) == result['length'] - assert 'columns' in result - assert 'data' in result - data = result['data'] - assert len(data) == 289 - # analyzed DF has 30 columns - assert len(result['columns']) == 30 - assert len(data[0]) == 30 - date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] - rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] + for call in ('get', 'post'): + if call == 'get': + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + f"&timerange=20180111-20180112&strategy={CURRENT_TEST_STRATEGY}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_history", + data={ + "pair": "UNITTEST/BTC", + "timeframe": timeframe, + "timerange": "20180111-20180112", + "strategy": CURRENT_TEST_STRATEGY, + "columns": ['rsi', 'fastd', 'fastk'], + }) - assert data[0][date_col_idx] == '2018-01-11T00:00:00Z' - assert data[0][rsi_col_idx] is not None - assert data[0][rsi_col_idx] > 0 - assert lfm.call_count == 1 - assert result['pair'] == 'UNITTEST/BTC' - assert result['strategy'] == CURRENT_TEST_STRATEGY - assert result['data_start'] == '2018-01-11 00:00:00+00:00' - assert result['data_start_ts'] == 1515628800000 - assert result['data_stop'] == '2018-01-12 00:00:00+00:00' - assert result['data_stop_ts'] == 1515715200000 + assert_response(rc, 200) + result = rc.json() + assert result['length'] == 289 + assert len(result['data']) == result['length'] + assert 'columns' in result + assert 'data' in result + data = result['data'] + assert len(data) == 289 + col_count = 30 if call == 'get' else 18 + # analyzed DF has 30 columns + assert len(result['columns']) == col_count + assert len(result['all_columns']) == 25 + assert len(data[0]) == col_count + date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] + rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] - # No data found - rc = client_get(client, - f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" - f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}") - assert_response(rc, 502) - assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.") + assert data[0][date_col_idx] == '2018-01-11T00:00:00Z' + assert data[0][rsi_col_idx] is not None + assert data[0][rsi_col_idx] > 0 + assert lfm.call_count == 1 + assert result['pair'] == 'UNITTEST/BTC' + assert result['strategy'] == CURRENT_TEST_STRATEGY + assert result['data_start'] == '2018-01-11 00:00:00+00:00' + assert result['data_start_ts'] == 1515628800000 + assert result['data_stop'] == '2018-01-12 00:00:00+00:00' + assert result['data_stop_ts'] == 1515715200000 + lfm.reset_mock() + + # No data found + if call == 'get': + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + f"&timerange=20200111-20200112&strategy={CURRENT_TEST_STRATEGY}") + else: + rc = client_post( + client, + f"{BASE_URI}/pair_history", + data={ + "pair": "UNITTEST/BTC", + "timeframe": timeframe, + "timerange": "20200111-20200112", + "strategy": CURRENT_TEST_STRATEGY, + "columns": ['rsi', 'fastd', 'fastk'], + }) + assert_response(rc, 502) + assert rc.json()['detail'] == ("No data for UNITTEST/BTC, 5m in 20200111-20200112 found.") def test_api_plot_config(botclient, mocker, tmp_path):