From 2df80fc49a2b8f320a5881fbae75dc4b75077f31 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 18:35:52 +0200 Subject: [PATCH 01/39] Add /pairlists endpoint to api --- freqtrade/rpc/api_server/api_schemas.py | 4 ++++ freqtrade/rpc/api_server/api_v1.py | 26 +++++++++++++++++++------ tests/rpc/test_rpc_apiserver.py | 16 +++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 53bf7558f..64d32b15f 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -389,6 +389,10 @@ class StrategyListResponse(BaseModel): strategies: List[str] +class PairListResponse(BaseModel): + pairlists: List[Dict[str, Any]] + + class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8aa706e62..7a3a5ca84 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,11 +15,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PerformanceEntry, - Ping, PlotConfig, Profit, ResultMsg, ShowConfig, - Stats, StatusMsg, StrategyListResponse, - StrategyResponse, SysInfo, Version, - WhitelistResponse) + OpenTradeSchema, PairHistory, PairListResponse, + PerformanceEntry, Ping, PlotConfig, Profit, + ResultMsg, ShowConfig, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, SysInfo, + Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -43,7 +43,8 @@ logger = logging.getLogger(__name__) # 2.23: Allow plot config request in webserver mode # 2.24: Add cancel_open_order endpoint # 2.25: Add several profit values to /status endpoint -API_VERSION = 2.25 +# 2.26: new /pairlists endpoint +API_VERSION = 2.26 # Public API, requires no auth. router_public = APIRouter() @@ -300,6 +301,19 @@ def get_strategy(strategy: str, config=Depends(get_config)): } +@router.get('/pairlists', response_model=PairListResponse) +def list_pairlists(config=Depends(get_config)): + from freqtrade.resolvers import PairListResolver + pairlists = PairListResolver.search_all_objects( + config, False) + pairlists = sorted(pairlists, key=lambda x: x['name']) + + return {'pairlists': [{ + "name": x['name'], + } for x in pairlists + ]} + + @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 58c904838..d5a375f19 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1580,6 +1580,22 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} +def test_api_pairlists(botclient, tmpdir, mocker): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists") + + assert_response(rc) + response = rc.json() + assert isinstance(response['pairlists'], list) + assert len(response['pairlists']) > 0 + + assert len([r for r in response['pairlists'] if r['name'] == 'AgeFilter']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'VolumePairList']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'StaticPairList']) == 1 + + def test_list_available_pairs(botclient): ftbot, client = botclient From 5ad352fdf1ab1bcc5c605768f0d55f24e86d80cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 21:08:28 +0200 Subject: [PATCH 02/39] add /pairlists to rest client --- scripts/rest_client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 196542780..3c9050f43 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -313,6 +313,13 @@ class FtRestClient(): """ return self._get(f"strategy/{strategy}") + def pairlists(self): + """Lists available pairlists + + :return: json object + """ + return self._get("pairlists") + def plot_config(self): """Return plot configuration if the strategy defines one. From 987da010c97c24b52034aa0092704720aabf828e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 21:08:44 +0200 Subject: [PATCH 03/39] Start pairlist parameter listing --- freqtrade/plugins/pairlist/AgeFilter.py | 25 ++++++++++++++++++++- freqtrade/plugins/pairlist/IPairList.py | 30 ++++++++++++++++++++++++- freqtrade/rpc/api_server/api_v1.py | 1 + 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f9c02e250..66556841b 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -12,7 +12,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import PeriodicCache @@ -68,6 +68,29 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + """ + Return parameters used by this Pairlist Handler, and their type + contains a dictionary with the parameter name as key, and a dictionary + with the type and default value. + -> Please overwrite in subclasses + """ + return { + "min_days_listed": { + "type": "number", + "default": 10, + "description": "Minimum Days Listed", + "help": "Minimum number of days a pair must have been listed on the exchange.", + }, + "max_days_listed": { + "type": "number", + "default": None, + "description": "Maximum Days Listed", + "help": "Maximum number of days a pair must have been listed on the exchange.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ :param pairlist: pairlist to filter or sort diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index d0382c778..3e143aa69 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -4,7 +4,7 @@ PairList Handler base class import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union from freqtrade.constants import Config from freqtrade.exceptions import OperationalException @@ -16,6 +16,13 @@ from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) +class PairlistParameter(TypedDict): + type: Literal["number", "string", "boolean"] + default: Union[int, float, str, bool] + description: str + help: str + + class IPairList(LoggingMixin, ABC): def __init__(self, exchange: Exchange, pairlistmanager, @@ -54,6 +61,27 @@ class IPairList(LoggingMixin, ABC): as tickers argument to filter_pairlist """ + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + """ + Return parameters used by this Pairlist Handler, and their type + contains a dictionary with the parameter name as key, and a dictionary + with the type and default value. + -> Please overwrite in subclasses + """ + return {} + + @staticmethod + def refresh_period(params: Dict[str, PairlistParameter]) -> None: + return { + "refresh_period": { + "type": "number", + "default": 1800, + "description": "Refresh period", + "help": "Refresh period in seconds", + } + } + @abstractmethod def short_desc(self) -> str: """ diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7a3a5ca84..357ccfd24 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -310,6 +310,7 @@ def list_pairlists(config=Depends(get_config)): return {'pairlists': [{ "name": x['name'], + "params": x['class'].available_parameters(), } for x in pairlists ]} From 2ea157d9d3507a3eb4f45ec806f903f9a58bc3d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 06:58:05 +0200 Subject: [PATCH 04/39] Add some more pairlist parameter definitions --- freqtrade/plugins/pairlist/AgeFilter.py | 6 ------ freqtrade/plugins/pairlist/IPairList.py | 4 ++-- freqtrade/plugins/pairlist/OffsetFilter.py | 19 ++++++++++++++++++- .../plugins/pairlist/PerformanceFilter.py | 19 ++++++++++++++++++- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- freqtrade/plugins/pairlist/StaticPairList.py | 13 ++++++++++++- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 66556841b..cd8fb9c68 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -70,12 +70,6 @@ class AgeFilter(IPairList): @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: - """ - Return parameters used by this Pairlist Handler, and their type - contains a dictionary with the parameter name as key, and a dictionary - with the type and default value. - -> Please overwrite in subclasses - """ return { "min_days_listed": { "type": "number", diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 3e143aa69..589240b10 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class PairlistParameter(TypedDict): type: Literal["number", "string", "boolean"] - default: Union[int, float, str, bool] + default: Union[int, float, str, bool, None] description: str help: str @@ -72,7 +72,7 @@ class IPairList(LoggingMixin, ABC): return {} @staticmethod - def refresh_period(params: Dict[str, PairlistParameter]) -> None: + def refresh_period_parameter() -> Dict[str, PairlistParameter]: return { "refresh_period": { "type": "number", diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 8f21cdd85..e7688a66c 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -43,6 +43,23 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "offset": { + "type": "number", + "default": 0, + "description": "Offset", + "help": "Offset of the pairlist.", + }, + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist, starting from offset.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index e7fcac1e4..6c582a4df 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -9,7 +9,7 @@ import pandas as pd from freqtrade.constants import Config from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -40,6 +40,23 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "minutes": { + "type": "number", + "default": 0, + "description": "Minutes", + "help": "Consider trades from the last X minutes. 0 means all trades.", + }, + "min_profit": { + "type": "number", + "default": None, + "description": "Minimum profit", + "help": "Minimum profit in percent. Pairs with less profit are removed.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the allowlist again. diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 2e74aa293..08f7ac5d6 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -8,7 +8,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import ROUND_UP from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 4b1961a53..f94c91d3d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -40,6 +40,17 @@ class StaticPairList(IPairList): """ return f"{self.name}" + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "allow_inactive": { + "type": "boolean", + "default": False, + "description": "Allow inactive pairs", + "help": "Allow inactive pairs to be in the whitelist.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist From 4636de30cd472321dcb8081de6f66f7e0ba568da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 07:03:27 +0200 Subject: [PATCH 05/39] Improve pairlistparam types --- freqtrade/plugins/pairlist/IPairList.py | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 589240b10..cadc706d6 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -16,13 +16,33 @@ from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) -class PairlistParameter(TypedDict): - type: Literal["number", "string", "boolean"] - default: Union[int, float, str, bool, None] +class __PairlistParameterBase(TypedDict): description: str help: str +class __NumberPairlistParameter(__PairlistParameterBase): + type: Literal["number"] + default: Union[int, float, None] + + +class __StringPairlistParameter(__PairlistParameterBase): + type: Literal["string"] + default: Union[str, None] + + +class __BoolPairlistParameter(__PairlistParameterBase): + type: Literal["boolean"] + default: Union[bool, None] + + +PairlistParameter = Union[ + __NumberPairlistParameter, + __StringPairlistParameter, + __BoolPairlistParameter + ] + + class IPairList(LoggingMixin, ABC): def __init__(self, exchange: Exchange, pairlistmanager, From e20b94d836b50bbf5930ee5cbdb3d9b0f1f60c41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 07:20:45 +0200 Subject: [PATCH 06/39] Add more filter param descriptions --- freqtrade/plugins/pairlist/PriceFilter.py | 32 +++++++++++++- .../plugins/pairlist/ProducerPairList.py | 20 ++++++++- freqtrade/plugins/pairlist/RemotePairList.py | 38 +++++++++++++++- freqtrade/plugins/pairlist/ShuffleFilter.py | 19 +++++++- freqtrade/plugins/pairlist/SpreadFilter.py | 13 +++++- .../plugins/pairlist/VolatilityFilter.py | 26 ++++++++++- freqtrade/plugins/pairlist/VolumePairList.py | 44 ++++++++++++++++++- .../plugins/pairlist/rangestabilityfilter.py | 26 ++++++++++- 8 files changed, 210 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 4d23de792..d87646f42 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -65,6 +65,36 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "low_price_ratio": { + "type": "number", + "default": 0, + "description": "Low price ratio", + "help": ("Remove pairs where a price move of 1 price unit (pip) " + "is above this ratio."), + }, + "min_price": { + "type": "number", + "default": 0, + "description": "Minimum price", + "help": "Remove pairs with a price below this value.", + }, + "max_price": { + "type": "number", + "default": 0, + "description": "Maximum price", + "help": "Remove pairs with a price above this value.", + }, + "max_value": { + "type": "number", + "default": 0, + "description": "Maximum value", + "help": "Remove pairs with a value (price * amount) above this value.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if if one price-step (pip) is > than a certain barrier. diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 882d49b76..1727efe92 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -56,6 +56,24 @@ class ProducerPairList(IPairList): """ return f"{self.name} - {self._producer_name}" + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "producer_name": { + "type": "string", + "default": "default", + "description": "Producer name", + "help": ("Name of the producer to use. Requires additional " + "external_message_consumer configuration.") + }, + } + def _filter_pairlist(self, pairlist: Optional[List[str]]): upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs( self._producer_name) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index d077330e0..a62745107 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -15,7 +15,7 @@ from freqtrade import __version__ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -63,6 +63,42 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist.", + }, + "pairlist_url": { + "type": "string", + "default": "", + "description": "URL to fetch pairlist from", + "help": "URL to fetch pairlist from", + }, + **IPairList.refresh_period_parameter(), + "keep_pairlist_on_failure": { + "type": "boolean", + "default": True, + "description": "Keep last pairlist on failure", + "help": "Keep last pairlist on failure", + }, + "read_timeout": { + "type": "number", + "default": 60, + "description": "Read timeout", + "help": "Request timeout for remote pairlist", + }, + "bearer_token": { + "type": "string", + "default": "", + "description": "Bearer token", + "help": "Bearer token - used for auth against the upstream service.", + }, + } + def process_json(self, jsonparse) -> List[str]: pairlist = jsonparse.get('pairs', []) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 76d7600d2..b566a0ac2 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -9,7 +9,7 @@ from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util.periodic_cache import PeriodicCache @@ -55,6 +55,23 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "shuffle_frequency": { + "type": "string", + "default": "candle", + "description": "Shuffle frequency", + "help": "Shuffle frequency. Can be either 'candle' or 'iteration'.", + }, + "seed": { + "type": "number", + "default": None, + "description": "Random Seed", + "help": "Seed for random number generator. Not used in live mode.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index d47b68568..9778c8072 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -45,6 +45,17 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "max_spread_ratio": { + "type": "number", + "default": 0.005, + "description": "Max spread ratio", + "help": "Max spread ratio for a pair to be considered.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Validate spread for the ticker diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 401a2e86c..59092d437 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -15,7 +15,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -63,6 +63,30 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_volatility": { + "type": "number", + "default": 0, + "description": "Minimum Volatility", + "help": "Minimum volatility a pair must have to be considered.", + }, + "max_volatility": { + "type": "number", + "default": None, + "description": "Maximum Volatility", + "help": "Maximum volatility a pair must have to be considered.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 2649a8425..4bf8ca74a 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -111,6 +111,48 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "sort_key": { + "type": "string", + "default": "quoteVolume", + "description": "Sort key", + "help": "Sort key to use for sorting the pairlist.", + }, + "min_value": { + "type": "number", + "default": 0, + "description": "Minimum value", + "help": "Minimum value to use for filtering the pairlist.", + }, + **IPairList.refresh_period_parameter(), + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "lookback_timeframe": { + "type": "string", + "default": "1d", + "description": "Lookback Timeframe", + "help": "Timeframe to use for lookback.", + }, + "lookback_period": { + "type": "number", + "default": 0, + "description": "Lookback Period", + "help": "Number of periods to look back at.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 546b026cb..fde3de1f0 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -13,7 +13,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -61,6 +61,30 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_rate_of_change": { + "type": "number", + "default": 0.01, + "description": "Minimum Rate of Change", + "help": "Minimum rate of change to filter pairs.", + }, + "max_rate_of_change": { + "type": "number", + "default": None, + "description": "Maximum Rate of Change", + "help": "Maximum rate of change to filter pairs.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range From 3ef2a57bcad765142880e904b206f1f33bc8b0db Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:09:47 +0200 Subject: [PATCH 07/39] Add "is_pairlist_generator" field to pairlists --- freqtrade/plugins/pairlist/IPairList.py | 2 ++ freqtrade/plugins/pairlist/ProducerPairList.py | 1 + freqtrade/plugins/pairlist/RemotePairList.py | 2 ++ freqtrade/plugins/pairlist/StaticPairList.py | 2 ++ freqtrade/plugins/pairlist/VolumePairList.py | 2 ++ 5 files changed, 9 insertions(+) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index cadc706d6..f752db3b1 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -45,6 +45,8 @@ PairlistParameter = Union[ class IPairList(LoggingMixin, ABC): + is_pairlist_generator = False + def __init__(self, exchange: Exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 1727efe92..623727671 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -28,6 +28,7 @@ class ProducerPairList(IPairList): } ], """ + is_pairlist_generator = True def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index a62745107..2cb157a97 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) class RemotePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index f94c91d3d..3601fb460 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -17,6 +17,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 4bf8ca74a..9bca900c7 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -25,6 +25,8 @@ SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: From 9e4f9798e628a37555089d0cf2371aa1756ca110 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:13:43 +0200 Subject: [PATCH 08/39] Add pairlist "is-generator" to api --- freqtrade/rpc/api_server/api_v1.py | 1 + tests/rpc/test_rpc_apiserver.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 357ccfd24..0b81ec6ae 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -310,6 +310,7 @@ def list_pairlists(config=Depends(get_config)): return {'pairlists': [{ "name": x['name'], + "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), } for x in pairlists ]} diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index d5a375f19..82c357fcb 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1595,6 +1595,13 @@ def test_api_pairlists(botclient, tmpdir, mocker): assert len([r for r in response['pairlists'] if r['name'] == 'VolumePairList']) == 1 assert len([r for r in response['pairlists'] if r['name'] == 'StaticPairList']) == 1 + volumepl = [r for r in response['pairlists'] if r['name'] == 'VolumePairList'][0] + assert volumepl['is_pairlist_generator'] is True + assert len(volumepl['params']) > 1 + age_pl = [r for r in response['pairlists'] if r['name'] == 'AgeFilter'][0] + assert age_pl['is_pairlist_generator'] is False + assert len(volumepl['params']) > 2 + def test_list_available_pairs(botclient): ftbot, client = botclient From c5bf029701d31b616cd425115ade8c19c2dd3a78 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:15:31 +0200 Subject: [PATCH 09/39] Better type response --- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- freqtrade/rpc/api_server/api_schemas.py | 8 +++++++- freqtrade/rpc/api_server/api_v1.py | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 08f7ac5d6..2e74aa293 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -8,7 +8,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import ROUND_UP from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 64d32b15f..13bc948d0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -390,7 +390,13 @@ class StrategyListResponse(BaseModel): class PairListResponse(BaseModel): - pairlists: List[Dict[str, Any]] + name: str + is_pairlist_generator: bool + params: Dict[str, Any] + + +class PairListsResponse(BaseModel): + pairlists: List[PairListResponse] class FreqAIModelListResponse(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 0b81ec6ae..9eeda451e 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,7 +15,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListResponse, + OpenTradeSchema, PairHistory, PairListsResponse, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, SysInfo, @@ -301,7 +301,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListResponse) +@router.get('/pairlists', response_model=PairListsResponse) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( From 3d4be92cc6567015fa0936b070eda8edf39ac32b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 21 Apr 2023 19:30:32 +0200 Subject: [PATCH 10/39] Add option pairlist parameter type --- freqtrade/plugins/pairlist/IPairList.py | 7 +++++++ freqtrade/plugins/pairlist/ShuffleFilter.py | 3 ++- freqtrade/plugins/pairlist/VolumePairList.py | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index f752db3b1..5f1b7e591 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -31,6 +31,12 @@ class __StringPairlistParameter(__PairlistParameterBase): default: Union[str, None] +class __OptionPairlistParameter(__PairlistParameterBase): + type: Literal["option"] + default: Union[str, None] + options: List[str] + + class __BoolPairlistParameter(__PairlistParameterBase): type: Literal["boolean"] default: Union[bool, None] @@ -39,6 +45,7 @@ class __BoolPairlistParameter(__PairlistParameterBase): PairlistParameter = Union[ __NumberPairlistParameter, __StringPairlistParameter, + __OptionPairlistParameter, __BoolPairlistParameter ] diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index b566a0ac2..5419295c9 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -59,8 +59,9 @@ class ShuffleFilter(IPairList): def available_parameters() -> Dict[str, PairlistParameter]: return { "shuffle_frequency": { - "type": "string", + "type": "option", "default": "candle", + "options": ["candle", "iteration"], "description": "Shuffle frequency", "help": "Shuffle frequency. Can be either 'candle' or 'iteration'.", }, diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 9bca900c7..c6d9902a4 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,8 +123,9 @@ class VolumePairList(IPairList): "help": "Number of assets to use from the pairlist", }, "sort_key": { - "type": "string", + "type": "option", "default": "quoteVolume", + "options": SORT_VALUES, "description": "Sort key", "help": "Sort key to use for sorting the pairlist.", }, From 877d53f4399d46ee62e768982bb6f02d94615e5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Apr 2023 20:35:24 +0200 Subject: [PATCH 11/39] Add airlists test endpoint (so pairlist configurations can be tested) --- freqtrade/rpc/api_server/api_schemas.py | 10 +++++++ freqtrade/rpc/api_server/api_v1.py | 38 +++++++++++++++++++++---- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 13bc948d0..0267ee82e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -399,6 +399,16 @@ class PairListsResponse(BaseModel): pairlists: List[PairListResponse] +class PairListsPayload(BaseModel): + pairlists: List[Dict[str, Any]] + blacklist: List[str] + stake_currency: str + + +class PairListsTest(BaseModel): + pairlists: List[PairListResponse] + + class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 9eeda451e..a8b9cfb27 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,11 +15,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListsResponse, - PerformanceEntry, Ping, PlotConfig, Profit, - ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistResponse) + OpenTradeSchema, PairHistory, PairListsPayload, + PairListsResponse, PerformanceEntry, Ping, + PlotConfig, Profit, ResultMsg, ShowConfig, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -301,7 +301,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListsResponse) +@router.get('/pairlists', response_model=PairListsResponse, tags=['pairlists', 'webserver']) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( @@ -316,6 +316,32 @@ def list_pairlists(config=Depends(get_config)): ]} +@router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) +def pairlists_test(payload: PairListsPayload, config=Depends(get_config)): + from freqtrade.plugins.pairlistmanager import PairListManager + from freqtrade.resolvers import ExchangeResolver + + config_loc = deepcopy(config) + + exchange = ExchangeResolver.load_exchange( + config_loc['exchange']['name'], config_loc, validate=False) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + + res = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + return res + + @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver From 680e7ba98ff1d3a22b00eca4dbd224955ad63efc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:21:22 +0200 Subject: [PATCH 12/39] Get exchange through DI --- freqtrade/rpc/api_server/api_v1.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 76dea3376..3633d584b 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -329,14 +329,11 @@ def list_pairlists(config=Depends(get_config)): @router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) -def pairlists_test(payload: PairListsPayload, config=Depends(get_config)): +def pairlists_test(payload: PairListsPayload, config=Depends(get_config), exchange=Depends(get_exchange)): from freqtrade.plugins.pairlistmanager import PairListManager - from freqtrade.resolvers import ExchangeResolver config_loc = deepcopy(config) - exchange = ExchangeResolver.load_exchange( - config_loc['exchange']['name'], config_loc, validate=False) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists From 818a3342b952fbf97da446d8d9dc7387ecd12539 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:38:14 +0200 Subject: [PATCH 13/39] move pairlist evaluation to the background --- freqtrade/rpc/api_server/api_v1.py | 67 ++++++++++++++------ freqtrade/rpc/api_server/webserver_bgwork.py | 3 + 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3633d584b..c61044b61 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -2,10 +2,11 @@ import logging from copy import deepcopy from typing import List, Optional -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException from freqtrade import __version__ +from freqtrade.constants import Config from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException @@ -21,6 +22,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac StatusMsg, StrategyListResponse, StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -313,7 +315,8 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListsResponse, tags=['pairlists', 'webserver']) +@router.get('/pairlists/available', + response_model=PairListsResponse, tags=['pairlists', 'webserver']) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( @@ -325,30 +328,54 @@ def list_pairlists(config=Depends(get_config)): "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), } for x in pairlists - ]} + ]} -@router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) -def pairlists_test(payload: PairListsPayload, config=Depends(get_config), exchange=Depends(get_exchange)): - from freqtrade.plugins.pairlistmanager import PairListManager +@router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) +def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, + config=Depends(get_config)): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + def run_pairlist(config_loc: Config): + try: + from freqtrade.plugins.pairlistmanager import PairListManager + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.pairlist_result = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + + finally: + ApiBG.pairlist_running = False config_loc = deepcopy(config) - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - - res = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist + background_tasks.add_task(run_pairlist, config_loc) + ApiBG.pairlist_running = True + return { + 'status': 'Pairlist evaluation started in background.' } - return res + + +@router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) +def pairlists_evaluate_get(): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + if not ApiBG.pairlist_result: + raise HTTPException(status_code=400, detail='Pairlist not started yet.') + + return ApiBG.pairlist_result @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 925f34de3..3ab36ced1 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -14,3 +14,6 @@ class ApiBG(): bgtask_running: bool = False # Exchange - only available in webserver mode. exchange = None + # Pairlist evaluate things + pairlist_running: bool = False + pairlist_result: Dict[str, Any] = {} From 7cc8da23c21c011c3fc0d03d6c2d72dc3ed95b99 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:56:46 +0200 Subject: [PATCH 14/39] Update test for available pairlist --- tests/rpc/test_rpc_apiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index a5b816f2a..e75d2cd71 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1616,11 +1616,11 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} -def test_api_pairlists(botclient, tmpdir, mocker): +def test_api_available_pairlists(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) - rc = client_get(client, f"{BASE_URI}/pairlists") + rc = client_get(client, f"{BASE_URI}/pairlists/available") assert_response(rc) response = rc.json() From 01984a06af849124fd5dc20beb45a28c22ef254c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:58:38 +0200 Subject: [PATCH 15/39] Extract pairlist evaluation from sub-method --- freqtrade/rpc/api_server/api_v1.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index c61044b61..d663b17d4 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -331,36 +331,37 @@ def list_pairlists(config=Depends(get_config)): ]} +def __run_pairlist(config_loc: Config): + try: + from freqtrade.plugins.pairlistmanager import PairListManager + + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.pairlist_result = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + + finally: + ApiBG.pairlist_running = False + + @router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') - def run_pairlist(config_loc: Config): - try: - from freqtrade.plugins.pairlistmanager import PairListManager - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.pairlist_result = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - - finally: - ApiBG.pairlist_running = False - config_loc = deepcopy(config) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist - background_tasks.add_task(run_pairlist, config_loc) + background_tasks.add_task(__run_pairlist, config_loc) ApiBG.pairlist_running = True return { 'status': 'Pairlist evaluation started in background.' @@ -370,7 +371,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: raise HTTPException(status_code=400, detail='Pairlist not started yet.') From 756e1f5d5bf64d2037d4c2ea192500b9bfa8f341 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 10:08:32 +0200 Subject: [PATCH 16/39] Test pairlist evaluation --- freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 54 +++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d663b17d4..52eab8754 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -374,7 +374,7 @@ def pairlists_evaluate_get(): raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: - raise HTTPException(status_code=400, detail='Pairlist not started yet.') + raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') return ApiBG.pairlist_result diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e75d2cd71..b33a093a6 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1616,7 +1616,7 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} -def test_api_available_pairlists(botclient, tmpdir): +def test_api_pairlists_available(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) @@ -1639,6 +1639,58 @@ def test_api_available_pairlists(botclient, tmpdir): assert len(volumepl['params']) > 2 +def test_api_pairlists_evaluate(botclient, tmpdir): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation not started yet.' + + ApiBG.pairlist_running = True + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' + + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC" + } + # Fail, already running + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation is already running.' + + # should start the run + ApiBG.pairlist_running = False + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc) + response = rc.json() + assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] + assert response['length'] == 4 + + # Restart with additional filter, reducing the list to 2 + body['pairlists'].append({"method": "OffsetFilter", "number_assets": 2}) + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc) + response = rc.json() + assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] + assert response['length'] == 2 + + def test_list_available_pairs(botclient): ftbot, client = botclient From 33e25434b4d43c0ce9d8d3d0c03ee9bcc80e091e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 May 2023 19:43:27 +0200 Subject: [PATCH 17/39] Change statuscode to 202 --- freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 52eab8754..54c3a1084 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -371,7 +371,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') + raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b33a093a6..6cd02f2c0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1650,7 +1650,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir): ApiBG.pairlist_running = True rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 400) + assert_response(rc, 202) assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' body = { From 4c52109fa34bfcdf86dd7ebca352c013a2b499d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 20:37:23 +0200 Subject: [PATCH 18/39] Handle pairlist evaluation errors gracefully --- freqtrade/rpc/api_server/api_v1.py | 12 ++++++++++-- freqtrade/rpc/api_server/webserver_bgwork.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 54c3a1084..411b622b4 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -4,6 +4,7 @@ from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException +from sympy import O from freqtrade import __version__ from freqtrade.constants import Config @@ -343,7 +344,9 @@ def __run_pairlist(config_loc: Config): 'length': len(pairlists.whitelist), 'whitelist': pairlists.whitelist } - + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.pairlist_error = str(e) finally: ApiBG.pairlist_running = False @@ -360,7 +363,8 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist - + ApiBG.pairlist_error = None + ApiBG.pairlist_result = {} background_tasks.add_task(__run_pairlist, config_loc) ApiBG.pairlist_running = True return { @@ -370,6 +374,10 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): + if ApiBG.pairlist_error: + raise HTTPException(status_code=500, + detail='Pairlist evaluation failed: ' + ApiBG.pairlist_error) + if ApiBG.pairlist_running: raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 3ab36ced1..1d2565265 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional class ApiBG(): @@ -15,5 +15,6 @@ class ApiBG(): # Exchange - only available in webserver mode. exchange = None # Pairlist evaluate things + pairlist_error: Optional[str] = None pairlist_running: bool = False pairlist_result: Dict[str, Any] = {} From 9e75c768c005beb2db351958d785b05f5d11c0d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 21:01:39 +0200 Subject: [PATCH 19/39] Improve responses for evaluate get endpoints --- freqtrade/rpc/api_server/api_schemas.py | 6 ++++++ freqtrade/rpc/api_server/api_v1.py | 22 ++++++++++++++-------- tests/rpc/test_rpc_apiserver.py | 16 ++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index ae9974348..e83e343dc 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -376,6 +376,12 @@ class WhitelistResponse(BaseModel): method: List[str] +class WhitelistEvaluateResponse(BaseModel): + result: Optional[WhitelistResponse] + error: Optional[str] + status: str + + class DeleteTrade(BaseModel): cancel_order_count: int result: str diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 411b622b4..3a6618291 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -21,7 +21,8 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac PairListsResponse, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, - SysInfo, Version, WhitelistResponse) + SysInfo, Version, WhitelistEvaluateResponse, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -372,19 +373,24 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa } -@router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) +@router.get('/pairlists/evaluate', response_model=WhitelistEvaluateResponse, tags=['pairlists']) def pairlists_evaluate_get(): - if ApiBG.pairlist_error: - raise HTTPException(status_code=500, - detail='Pairlist evaluation failed: ' + ApiBG.pairlist_error) if ApiBG.pairlist_running: - raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') + return {'status': 'running'} + if ApiBG.pairlist_error: + return { + 'status': 'failed', + 'error': ApiBG.pairlist_error + } if not ApiBG.pairlist_result: - raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') + return {'status': 'pending'} - return ApiBG.pairlist_result + return { + 'status': 'success', + 'result': ApiBG.pairlist_result + } @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6cd02f2c0..115d9ddb2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1645,13 +1645,13 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 400) - assert rc.json()['detail'] == 'Pairlist evaluation not started yet.' + assert_response(rc) + assert rc.json()['status'] == 'pending' ApiBG.pairlist_running = True rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 202) - assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' + assert_response(rc) + assert rc.json()['status'] == 'running' body = { "pairlists": [ @@ -1675,8 +1675,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") assert_response(rc) response = rc.json() - assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] - assert response['length'] == 4 + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] + assert response['result']['length'] == 4 # Restart with additional filter, reducing the list to 2 body['pairlists'].append({"method": "OffsetFilter", "number_assets": 2}) @@ -1687,8 +1687,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") assert_response(rc) response = rc.json() - assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] - assert response['length'] == 2 + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] + assert response['result']['length'] == 2 def test_list_available_pairs(botclient): From af7afa80a9546290e5012f3ff2fcbc05e3eecabd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 26 May 2023 06:44:48 +0200 Subject: [PATCH 20/39] remove gone-wrong import --- freqtrade/rpc/api_server/api_v1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3a6618291..bc78e4201 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -4,7 +4,6 @@ from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException -from sympy import O from freqtrade import __version__ from freqtrade.constants import Config From 1317de8c1c128c8be0c3f7e16453988a24c873b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 18:21:23 +0200 Subject: [PATCH 21/39] Add rudimentary description per pairlist --- freqtrade/plugins/pairlist/AgeFilter.py | 4 ++++ freqtrade/plugins/pairlist/IPairList.py | 10 ++++++++++ freqtrade/plugins/pairlist/OffsetFilter.py | 4 ++++ freqtrade/plugins/pairlist/PerformanceFilter.py | 4 ++++ freqtrade/plugins/pairlist/PrecisionFilter.py | 4 ++++ freqtrade/plugins/pairlist/PriceFilter.py | 4 ++++ freqtrade/plugins/pairlist/ProducerPairList.py | 4 ++++ freqtrade/plugins/pairlist/RemotePairList.py | 4 ++++ freqtrade/plugins/pairlist/ShuffleFilter.py | 4 ++++ freqtrade/plugins/pairlist/SpreadFilter.py | 4 ++++ freqtrade/plugins/pairlist/StaticPairList.py | 4 ++++ freqtrade/plugins/pairlist/VolatilityFilter.py | 4 ++++ freqtrade/plugins/pairlist/VolumePairList.py | 4 ++++ freqtrade/plugins/pairlist/rangestabilityfilter.py | 4 ++++ freqtrade/rpc/api_server/api_schemas.py | 5 +---- freqtrade/rpc/api_server/api_v1.py | 1 + 16 files changed, 64 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 014e5fb0b..bce789446 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -68,6 +68,10 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') + @staticmethod + def description() -> str: + return "Filter pairs by age (days listed)." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 5f1b7e591..d09b447d4 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -89,6 +89,16 @@ class IPairList(LoggingMixin, ABC): If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ + return False + + @staticmethod + @abstractmethod + def description() -> str: + """ + Return description of this Pairlist Handler + -> Please overwrite in subclasses + """ + return "" @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index e7688a66c..af152c7bc 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -43,6 +43,10 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." + @staticmethod + def description() -> str: + return "Offset pair list filter." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 6c582a4df..06c504317 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -40,6 +40,10 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." + @staticmethod + def description() -> str: + return "Filter pairs by performance." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 2e74aa293..d354eaf63 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -46,6 +46,10 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." + @staticmethod + def description() -> str: + return "Filters low-value coins which would not allow setting stoplosses." + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index d87646f42..4c8781184 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -65,6 +65,10 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." + @staticmethod + def description() -> str: + return "Filter pairs by price." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 623727671..826f05913 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -57,6 +57,10 @@ class ProducerPairList(IPairList): """ return f"{self.name} - {self._producer_name}" + @staticmethod + def description() -> str: + return "Get a pairlist from an upstream bot." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 2cb157a97..372f9a593 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -65,6 +65,10 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + @staticmethod + def description() -> str: + return "Retrieve pairs from a remote API." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 5419295c9..ce37dd8b5 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -55,6 +55,10 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) + @staticmethod + def description() -> str: + return "Randomize pairlist order." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 9778c8072..ee41cbe66 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -45,6 +45,10 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") + @staticmethod + def description() -> str: + return "Filter by bid/ask difference." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 3601fb460..16fb97adb 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -42,6 +42,10 @@ class StaticPairList(IPairList): """ return f"{self.name}" + @staticmethod + def description() -> str: + return "Use pairlist as configured in config." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 6f7b9f7bc..800bf3664 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -64,6 +64,10 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filter pairs by their recent volatility." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 402092db3..e9913ad20 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -114,6 +114,10 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + @staticmethod + def description() -> str: + return "Provides dynamic pair list based on trade volumes." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index d227a284e..f294b882b 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -62,6 +62,10 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filters pairs by their rate of change." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index e83e343dc..d5e4b05aa 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -404,6 +404,7 @@ class StrategyListResponse(BaseModel): class PairListResponse(BaseModel): name: str + description: str is_pairlist_generator: bool params: Dict[str, Any] @@ -418,10 +419,6 @@ class PairListsPayload(BaseModel): stake_currency: str -class PairListsTest(BaseModel): - pairlists: List[PairListResponse] - - class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index bc78e4201..8e2926ea7 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -328,6 +328,7 @@ def list_pairlists(config=Depends(get_config)): "name": x['name'], "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), + "description": x['class'].description(), } for x in pairlists ]} From 6315516d50e901da64949d92111dc4034a8fa817 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 May 2023 15:18:46 +0200 Subject: [PATCH 22/39] Improve volumepairlist defaults --- freqtrade/plugins/pairlist/VolumePairList.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index e9913ad20..0d5e33847 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,7 +123,7 @@ class VolumePairList(IPairList): return { "number_assets": { "type": "number", - "default": 0, + "default": 30, "description": "Number of assets", "help": "Number of assets to use from the pairlist", }, @@ -143,13 +143,13 @@ class VolumePairList(IPairList): **IPairList.refresh_period_parameter(), "lookback_days": { "type": "number", - "default": 10, + "default": 0, "description": "Lookback Days", "help": "Number of days to look back at.", }, "lookback_timeframe": { "type": "string", - "default": "1d", + "default": "", "description": "Lookback Timeframe", "help": "Timeframe to use for lookback.", }, From 7bccf2129fcabd2a086c670b4dba11e2fb33b51e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 07:00:20 +0200 Subject: [PATCH 23/39] Introduce background_job endpoints --- freqtrade/rpc/api_server/api_schemas.py | 19 ++++- freqtrade/rpc/api_server/api_v1.py | 89 ++++++++++++++------ freqtrade/rpc/api_server/webserver_bgwork.py | 23 ++++- 3 files changed, 97 insertions(+), 34 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index d5e4b05aa..6ab848a50 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -27,6 +27,21 @@ class StatusMsg(BaseModel): status: str +class BgJobStarted(StatusMsg): + job_id: str + + +class BackgroundTaskStatus(BaseModel): + status: str + running: bool + progress: Optional[float] + + +class BackgroundTaskResult(BaseModel): + error: Optional[str] + status: str + + class ResultMsg(BaseModel): result: str @@ -376,10 +391,8 @@ class WhitelistResponse(BaseModel): method: List[str] -class WhitelistEvaluateResponse(BaseModel): +class WhitelistEvaluateResponse(BackgroundTaskResult): result: Optional[WhitelistResponse] - error: Optional[str] - status: str class DeleteTrade(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8e2926ea7..e9bc128ab 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -11,16 +11,17 @@ from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC -from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, - BlacklistResponse, Count, Daily, - DeleteLockRequest, DeleteTrade, ForceEnterPayload, - ForceEnterResponse, ForceExitPayload, - FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListsPayload, - PairListsResponse, PerformanceEntry, Ping, - PlotConfig, Profit, ResultMsg, ShowConfig, Stats, - StatusMsg, StrategyListResponse, StrategyResponse, - SysInfo, Version, WhitelistEvaluateResponse, +from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, BackgroundTaskStatus, Balances, + BgJobStarted, BlacklistPayload, BlacklistResponse, + Count, Daily, DeleteLockRequest, DeleteTrade, + ForceEnterPayload, ForceEnterResponse, + ForceExitPayload, FreqAIModelListResponse, Health, + Locks, Logs, OpenTradeSchema, PairHistory, + PairListsPayload, PairListsResponse, + PerformanceEntry, Ping, PlotConfig, Profit, + ResultMsg, ShowConfig, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, SysInfo, + Version, WhitelistEvaluateResponse, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.api_server.webserver_bgwork import ApiBG @@ -333,26 +334,30 @@ def list_pairlists(config=Depends(get_config)): ]} -def __run_pairlist(config_loc: Config): +def __run_pairlist(job_id: str, config_loc: Config): try: + + ApiBG.jobs[job_id]['is_running'] = True from freqtrade.plugins.pairlistmanager import PairListManager exchange = get_exchange(config_loc) pairlists = PairListManager(exchange, config_loc) pairlists.refresh_pairlist() - ApiBG.pairlist_result = { + ApiBG.jobs[job_id]['result'] = { 'method': pairlists.name_list, 'length': len(pairlists.whitelist), 'whitelist': pairlists.whitelist } + ApiBG.jobs[job_id]['status'] = 'success' except (OperationalException, Exception) as e: logger.exception(e) - ApiBG.pairlist_error = str(e) + ApiBG.jobs[job_id]['error'] = str(e) finally: + ApiBG.jobs[job_id]['is_running'] = False ApiBG.pairlist_running = False -@router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: @@ -364,32 +369,60 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist - ApiBG.pairlist_error = None - ApiBG.pairlist_result = {} - background_tasks.add_task(__run_pairlist, config_loc) + # Random job id + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + 'category': 'pairlist', + 'status': 'pending', + 'progress': None, + 'is_running': False, + 'result': {}, + 'error': None, + } + ApiBG.running_jobs.append(job_id) + background_tasks.add_task(__run_pairlist, job_id, config_loc) ApiBG.pairlist_running = True + return { - 'status': 'Pairlist evaluation started in background.' + 'status': 'Pairlist evaluation started in background.', + 'job_id': job_id, } -@router.get('/pairlists/evaluate', response_model=WhitelistEvaluateResponse, tags=['pairlists']) -def pairlists_evaluate_get(): +@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, + tags=['pairlists']) +def pairlists_evaluate_get(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') - if ApiBG.pairlist_running: - return {'status': 'running'} - if ApiBG.pairlist_error: + if job['is_running']: + raise HTTPException(status_code=400, detail='Job not finished yet.') + + if error := job['error']: return { 'status': 'failed', - 'error': ApiBG.pairlist_error + 'error': error, } - if not ApiBG.pairlist_result: - return {'status': 'pending'} - return { 'status': 'success', - 'result': ApiBG.pairlist_result + 'result': job['result'], + } + + +@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) +def background_job(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + return { + 'job_id': jobid, + # 'type': job['job_type'], + 'status': job['status'], + 'running': job['is_running'], + 'progress': job.get('progress'), + # 'job_error': job['error'], } diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 1d2565265..d4c36536c 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,5 +1,15 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Literal, Optional, TypedDict +from uuid import uuid4 + + +class JobsContainer(TypedDict): + category: Literal['pairlist'] + is_running: bool + status: str + progress: Optional[float] + result: Any + error: Optional[str] class ApiBG(): @@ -14,7 +24,14 @@ class ApiBG(): bgtask_running: bool = False # Exchange - only available in webserver mode. exchange = None + + # Generic background jobs + running_jobs: List[str] = [] + # TODO: Change this to TTLCache + jobs: Dict[str, JobsContainer] = {} # Pairlist evaluate things - pairlist_error: Optional[str] = None pairlist_running: bool = False - pairlist_result: Dict[str, Any] = {} + + @staticmethod + def get_job_id() -> str: + return str(uuid4()) From fd955028a87a125cab94deea1b1e7f6c00c60765 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 07:08:27 +0200 Subject: [PATCH 24/39] Update tests for new background method --- freqtrade/rpc/api_server/api_schemas.py | 2 ++ freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 30 ++++++++++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 6ab848a50..c3f31b3c6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -32,6 +32,8 @@ class BgJobStarted(StatusMsg): class BackgroundTaskStatus(BaseModel): + job_id: str + job_category: str status: str running: bool progress: Optional[float] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index e9bc128ab..7fb4ee248 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -418,7 +418,7 @@ def background_job(jobid: str): return { 'job_id': jobid, - # 'type': job['job_type'], + 'job_category': job['category'], 'status': job['status'], 'running': job['is_running'], 'progress': job.get('progress'), diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 115d9ddb2..476a8ff2d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1643,15 +1643,10 @@ def test_api_pairlists_evaluate(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") - assert_response(rc) - assert rc.json()['status'] == 'pending' - - ApiBG.pairlist_running = True - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc) - assert rc.json()['status'] == 'running' + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' body = { "pairlists": [ @@ -1662,6 +1657,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir): "stake_currency": "BTC" } # Fail, already running + ApiBG.pairlist_running = True rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc, 400) assert rc.json()['detail'] == 'Pairlist evaluation is already running.' @@ -1671,8 +1667,19 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) assert rc.json()['status'] == 'Pairlist evaluation started in background.' + job_id = rc.json()['job_id'] - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/background/RandomJob") + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' + + rc = client_get(client, f"{BASE_URI}/background/{job_id}") + assert_response(rc) + response = rc.json() + assert response['job_id'] == job_id + assert response['job_category'] == 'pairlist' + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") assert_response(rc) response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] @@ -1683,8 +1690,9 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) assert rc.json()['status'] == 'Pairlist evaluation started in background.' - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + job_id = rc.json()['job_id'] + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") assert_response(rc) response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] From 88ecb935b978eb738b5ea2d656fa07345813f00c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 20:22:22 +0200 Subject: [PATCH 25/39] Add "failed" state to bgjob task response --- freqtrade/rpc/api_server/api_v1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7fb4ee248..10247a1cc 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -354,6 +354,7 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.jobs[job_id]['error'] = str(e) finally: ApiBG.jobs[job_id]['is_running'] = False + ApiBG.jobs[job_id]['status'] = 'failed' ApiBG.pairlist_running = False From 77e3e9e899ea97eeacb57c0597c6838c991ec719 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Jun 2023 20:40:12 +0200 Subject: [PATCH 26/39] Move pairlists and background tasks API's to separate file --- .../rpc/api_server/api_background_tasks.py | 129 +++++++++++++++++ freqtrade/rpc/api_server/api_v1.py | 134 ++---------------- freqtrade/rpc/api_server/webserver.py | 5 + 3 files changed, 144 insertions(+), 124 deletions(-) create mode 100644 freqtrade/rpc/api_server/api_background_tasks.py diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py new file mode 100644 index 000000000..4cf2f7994 --- /dev/null +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -0,0 +1,129 @@ +import logging +from copy import deepcopy + +from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi.exceptions import HTTPException + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, + PairListsPayload, PairListsResponse, + WhitelistEvaluateResponse) +from freqtrade.rpc.api_server.deps import get_config, get_exchange +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG + + +logger = logging.getLogger(__name__) + +# Private API, protected by authentication and webserver_mode dependency +router = APIRouter() + + +@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) +def background_job(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + return { + 'job_id': jobid, + 'job_category': job['category'], + 'status': job['status'], + 'running': job['is_running'], + 'progress': job.get('progress'), + # 'job_error': job['error'], + } + + +@router.get('/pairlists/available', + response_model=PairListsResponse, tags=['pairlists', 'webserver']) +def list_pairlists(config=Depends(get_config)): + from freqtrade.resolvers import PairListResolver + pairlists = PairListResolver.search_all_objects( + config, False) + pairlists = sorted(pairlists, key=lambda x: x['name']) + + return {'pairlists': [{ + "name": x['name'], + "is_pairlist_generator": x['class'].is_pairlist_generator, + "params": x['class'].available_parameters(), + "description": x['class'].description(), + } for x in pairlists + ]} + + +def __run_pairlist(job_id: str, config_loc: Config): + try: + + ApiBG.jobs[job_id]['is_running'] = True + from freqtrade.plugins.pairlistmanager import PairListManager + + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.jobs[job_id]['error'] = str(e) + finally: + ApiBG.jobs[job_id]['is_running'] = False + ApiBG.jobs[job_id]['status'] = 'failed' + ApiBG.pairlist_running = False + + +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) +def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, + config=Depends(get_config)): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + config_loc = deepcopy(config) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + # Random job id + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + 'category': 'pairlist', + 'status': 'pending', + 'progress': None, + 'is_running': False, + 'result': {}, + 'error': None, + } + ApiBG.running_jobs.append(job_id) + background_tasks.add_task(__run_pairlist, job_id, config_loc) + ApiBG.pairlist_running = True + + return { + 'status': 'Pairlist evaluation started in background.', + 'job_id': job_id, + } + + +@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, + tags=['pairlists']) +def pairlists_evaluate_get(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + if job['is_running']: + raise HTTPException(status_code=400, detail='Job not finished yet.') + + if error := job['error']: + return { + 'status': 'failed', + 'error': error, + } + + return { + 'status': 'success', + 'result': job['result'], + } diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 10247a1cc..c49c2b333 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -2,29 +2,25 @@ import logging from copy import deepcopy from typing import List, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, Query +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException from freqtrade import __version__ -from freqtrade.constants import Config from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC -from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, BackgroundTaskStatus, Balances, - BgJobStarted, BlacklistPayload, BlacklistResponse, - Count, Daily, DeleteLockRequest, DeleteTrade, - ForceEnterPayload, ForceEnterResponse, - ForceExitPayload, FreqAIModelListResponse, Health, - Locks, Logs, OpenTradeSchema, PairHistory, - PairListsPayload, PairListsResponse, - PerformanceEntry, Ping, PlotConfig, Profit, - ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistEvaluateResponse, +from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, + BlacklistResponse, Count, Daily, + DeleteLockRequest, DeleteTrade, ForceEnterPayload, + ForceEnterResponse, ForceExitPayload, + FreqAIModelListResponse, Health, Locks, Logs, + OpenTradeSchema, PairHistory, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, ShowConfig, + Stats, StatusMsg, StrategyListResponse, + StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional -from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -317,116 +313,6 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists/available', - response_model=PairListsResponse, tags=['pairlists', 'webserver']) -def list_pairlists(config=Depends(get_config)): - from freqtrade.resolvers import PairListResolver - pairlists = PairListResolver.search_all_objects( - config, False) - pairlists = sorted(pairlists, key=lambda x: x['name']) - - return {'pairlists': [{ - "name": x['name'], - "is_pairlist_generator": x['class'].is_pairlist_generator, - "params": x['class'].available_parameters(), - "description": x['class'].description(), - } for x in pairlists - ]} - - -def __run_pairlist(job_id: str, config_loc: Config): - try: - - ApiBG.jobs[job_id]['is_running'] = True - from freqtrade.plugins.pairlistmanager import PairListManager - - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.jobs[job_id]['result'] = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - ApiBG.jobs[job_id]['status'] = 'success' - except (OperationalException, Exception) as e: - logger.exception(e) - ApiBG.jobs[job_id]['error'] = str(e) - finally: - ApiBG.jobs[job_id]['is_running'] = False - ApiBG.jobs[job_id]['status'] = 'failed' - ApiBG.pairlist_running = False - - -@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) -def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, - config=Depends(get_config)): - if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') - - config_loc = deepcopy(config) - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - # Random job id - job_id = ApiBG.get_job_id() - - ApiBG.jobs[job_id] = { - 'category': 'pairlist', - 'status': 'pending', - 'progress': None, - 'is_running': False, - 'result': {}, - 'error': None, - } - ApiBG.running_jobs.append(job_id) - background_tasks.add_task(__run_pairlist, job_id, config_loc) - ApiBG.pairlist_running = True - - return { - 'status': 'Pairlist evaluation started in background.', - 'job_id': job_id, - } - - -@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, - tags=['pairlists']) -def pairlists_evaluate_get(jobid: str): - if not (job := ApiBG.jobs.get(jobid)): - raise HTTPException(status_code=404, detail='Job not found.') - - if job['is_running']: - raise HTTPException(status_code=400, detail='Job not finished yet.') - - if error := job['error']: - return { - 'status': 'failed', - 'error': error, - } - - return { - 'status': 'success', - 'result': job['result'], - } - - -@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) -def background_job(jobid: str): - if not (job := ApiBG.jobs.get(jobid)): - raise HTTPException(status_code=404, detail='Job not found.') - - return { - 'job_id': jobid, - 'job_category': job['category'], - 'status': job['status'], - 'running': job['is_running'], - 'progress': job.get('progress'), - # 'job_error': job['error'], - } - - @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index ea623e0ed..4d934eee3 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -114,6 +114,7 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login + from freqtrade.rpc.api_server.api_background_tasks import router as api_bg_tasks from freqtrade.rpc.api_server.api_backtest import router as api_backtest from freqtrade.rpc.api_server.api_v1 import router as api_v1 from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public @@ -130,6 +131,10 @@ class ApiServer(RPCHandler): dependencies=[Depends(http_basic_or_jwt_token), Depends(is_webserver_mode)], ) + app.include_router(api_bg_tasks, prefix="/api/v1", + dependencies=[Depends(http_basic_or_jwt_token), + Depends(is_webserver_mode)], + ) app.include_router(ws_router, prefix="/api/v1") app.include_router(router_login, prefix="/api/v1", tags=["auth"]) # UI Router MUST be last! From e2594e7494418797d675a10864d49a8deebcea1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Jun 2023 20:46:28 +0200 Subject: [PATCH 27/39] Align tests to use webserver mode --- tests/rpc/test_rpc_apiserver.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9c5891b03..9d0193d4a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1622,6 +1622,12 @@ def test_api_pairlists_available(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/available") + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/available") assert_response(rc) response = rc.json() assert isinstance(response['pairlists'], list) @@ -1645,6 +1651,12 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") assert_response(rc, 404) assert rc.json()['detail'] == 'Job not found.' From af16ce874c2bfe53b73f886bce04cf083fe68017 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Jun 2023 09:50:40 +0200 Subject: [PATCH 28/39] Allow webserver mode to cache multiple exchanges --- freqtrade/rpc/api_server/api_background_tasks.py | 1 - freqtrade/rpc/api_server/deps.py | 16 +++++++++++++--- freqtrade/rpc/api_server/webserver_bgwork.py | 8 +++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 4cf2f7994..3aa432995 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -98,7 +98,6 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa 'result': {}, 'error': None, } - ApiBG.running_jobs.append(job_id) background_tasks.add_task(__run_pairlist, job_id, config_loc) ApiBG.pairlist_running = True diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index 4c118d274..bface89bd 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -3,6 +3,7 @@ from uuid import uuid4 from fastapi import Depends, HTTPException +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.persistence import Trade from freqtrade.persistence.models import _request_id_ctx_var @@ -43,12 +44,21 @@ def get_api_config() -> Dict[str, Any]: return ApiServer._config['api_server'] +def _generate_exchange_key(config: Config) -> str: + """ + Exchange key - used for caching the exchange object. + """ + return f"{config['exchange']['name']}_{config.get('trading_mode', 'spot')}" + + def get_exchange(config=Depends(get_config)): - if not ApiBG.exchange: + exchange_key = _generate_exchange_key(config) + if not (exchange := ApiBG.exchanges.get(exchange_key)): from freqtrade.resolvers import ExchangeResolver - ApiBG.exchange = ExchangeResolver.load_exchange( + exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiBG.exchange + ApiBG.exchanges[exchange_key] = exchange + return exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index d4c36536c..3846fe138 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,7 +1,9 @@ -from typing import Any, Dict, List, Literal, Optional, TypedDict +from typing import Any, Dict, Literal, Optional, TypedDict from uuid import uuid4 +from freqtrade.exchange.exchange import Exchange + class JobsContainer(TypedDict): category: Literal['pairlist'] @@ -23,10 +25,10 @@ class ApiBG(): } bgtask_running: bool = False # Exchange - only available in webserver mode. - exchange = None + exchanges: Dict[str, Exchange] = {} # Generic background jobs - running_jobs: List[str] = [] + # TODO: Change this to TTLCache jobs: Dict[str, JobsContainer] = {} # Pairlist evaluate things From ac046d6a2da758a9cd0edfc82b80ae67005d8994 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Jun 2023 10:14:11 +0200 Subject: [PATCH 29/39] Allow setting the exchange explicitly --- .../rpc/api_server/api_background_tasks.py | 4 ++++ freqtrade/rpc/api_server/api_schemas.py | 2 ++ tests/rpc/test_rpc_apiserver.py | 20 ++++++++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 3aa432995..8538f1b9a 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -84,6 +84,10 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc = deepcopy(config) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists + if payload.exchange: + config_loc['exchange']['name'] = payload.exchange + if payload.trading_mode: + config_loc['trading_mode'] = payload.trading_mode # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index c3f31b3c6..1f5a79aef 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -432,6 +432,8 @@ class PairListsPayload(BaseModel): pairlists: List[Dict[str, Any]] blacklist: List[str] stake_currency: str + trading_mode: Optional[TradingMode] + exchange: Optional[str] class FreqAIModelListResponse(BaseModel): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9d0193d4a..f4229e0a0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1645,7 +1645,7 @@ def test_api_pairlists_available(botclient, tmpdir): assert len(volumepl['params']) > 2 -def test_api_pairlists_evaluate(botclient, tmpdir): +def test_api_pairlists_evaluate(botclient, tmpdir, mocker): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) @@ -1709,6 +1709,24 @@ def test_api_pairlists_evaluate(botclient, tmpdir): response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] assert response['result']['length'] == 2 + # Patch __run_pairlists + plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', return_value=None) + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC", + "exchange": "randomExchange", + "trading_mode": "futures", + } + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert plm.call_count == 1 + call_config = plm.call_args_list[0][0][1] + assert call_config['exchange']['name'] == 'randomExchange' + assert call_config['trading_mode'] == 'futures' def test_list_available_pairs(botclient): From 48328fb29de63bd62d0aa563ecc2e116a2d1c848 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:52:25 +0200 Subject: [PATCH 30/39] reset candle_type_def --- freqtrade/rpc/api_server/api_background_tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 8538f1b9a..8a6814801 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends from fastapi.exceptions import HTTPException from freqtrade.constants import Config +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, PairListsPayload, PairListsResponse, @@ -88,6 +89,8 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc['exchange']['name'] = payload.exchange if payload.trading_mode: config_loc['trading_mode'] = payload.trading_mode + config_loc['candle_type_def'] = CandleType.get_default( + config_loc.get('trading_mode', 'spot') or 'spot') # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist From d9d1735333d0781a2b52f0ad4b4c9f066dba7bc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:57:25 +0200 Subject: [PATCH 31/39] Extract ExchangePayload updating --- .../rpc/api_server/api_background_tasks.py | 24 ++++++++++++------- freqtrade/rpc/api_server/api_schemas.py | 9 ++++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 8a6814801..504ae7837 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -8,8 +8,8 @@ from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, - PairListsPayload, PairListsResponse, - WhitelistEvaluateResponse) + ExchangeModePayloadMixin, PairListsPayload, + PairListsResponse, WhitelistEvaluateResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange from freqtrade.rpc.api_server.webserver_bgwork import ApiBG @@ -85,12 +85,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc = deepcopy(config) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists - if payload.exchange: - config_loc['exchange']['name'] = payload.exchange - if payload.trading_mode: - config_loc['trading_mode'] = payload.trading_mode - config_loc['candle_type_def'] = CandleType.get_default( - config_loc.get('trading_mode', 'spot') or 'spot') + handleExchangePayload(payload, config_loc) # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist @@ -114,6 +109,19 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa } +def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config): + """ + Handle exchange and trading mode payload. + Updates the configuration with the payload values. + """ + if payload.exchange: + config_loc['exchange']['name'] = payload.exchange + if payload.trading_mode: + config_loc['trading_mode'] = payload.trading_mode + config_loc['candle_type_def'] = CandleType.get_default( + config_loc.get('trading_mode', 'spot') or 'spot') + + @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, tags=['pairlists']) def pairlists_evaluate_get(jobid: str): diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 1f5a79aef..b849ebeb7 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -7,6 +7,11 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode +class ExchangeModePayloadMixin(BaseModel): + trading_mode: Optional[TradingMode] + exchange: Optional[str] + + class Ping(BaseModel): status: str @@ -428,12 +433,10 @@ class PairListsResponse(BaseModel): pairlists: List[PairListResponse] -class PairListsPayload(BaseModel): +class PairListsPayload(ExchangeModePayloadMixin, BaseModel): pairlists: List[Dict[str, Any]] blacklist: List[str] stake_currency: str - trading_mode: Optional[TradingMode] - exchange: Optional[str] class FreqAIModelListResponse(BaseModel): From 10ea2b44c7753902a7a87ad43d29add7397f8868 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:59:22 +0200 Subject: [PATCH 32/39] Update test line length --- tests/rpc/test_rpc_apiserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f4229e0a0..7c815d8eb 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1710,7 +1710,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] assert response['result']['length'] == 2 # Patch __run_pairlists - plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', return_value=None) + plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', + return_value=None) body = { "pairlists": [ {"method": "StaticPairList", }, From 71b81ee7cd9371b5696f05ba021e6d4ec42c5e82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jun 2023 13:25:39 +0200 Subject: [PATCH 33/39] Add margin_mode to pairlists callback --- freqtrade/enums/marginmode.py | 2 +- freqtrade/rpc/api_server/api_background_tasks.py | 2 ++ freqtrade/rpc/api_server/api_schemas.py | 3 ++- tests/rpc/test_rpc_apiserver.py | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/enums/marginmode.py b/freqtrade/enums/marginmode.py index 1e42809ea..7fd749b29 100644 --- a/freqtrade/enums/marginmode.py +++ b/freqtrade/enums/marginmode.py @@ -1,7 +1,7 @@ from enum import Enum -class MarginMode(Enum): +class MarginMode(str, Enum): """ Enum to distinguish between cross margin/futures margin_mode and diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 504ae7837..158a89385 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -120,6 +120,8 @@ def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config) config_loc['trading_mode'] = payload.trading_mode config_loc['candle_type_def'] = CandleType.get_default( config_loc.get('trading_mode', 'spot') or 'spot') + if payload.margin_mode: + config_loc['margin_mode'] = payload.margin_mode @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 9b662b342..3f4dd99e1 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -4,12 +4,13 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf -from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode +from freqtrade.enums import MarginMode, OrderTypeValues, SignalDirection, TradingMode from freqtrade.types import ValidExchangesType class ExchangeModePayloadMixin(BaseModel): trading_mode: Optional[TradingMode] + margin_mode: Optional[MarginMode] exchange: Optional[str] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 88ba9d7e0..f793b1f9c 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1762,6 +1762,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): "stake_currency": "BTC", "exchange": "randomExchange", "trading_mode": "futures", + "margin_mode": "isolated", } rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) @@ -1769,6 +1770,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): call_config = plm.call_args_list[0][0][1] assert call_config['exchange']['name'] == 'randomExchange' assert call_config['trading_mode'] == 'futures' + assert call_config['margin_mode'] == 'isolated' def test_list_available_pairs(botclient): From 9ef814689e184f2425d51221107aae8d5fcac36d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:18:01 +0200 Subject: [PATCH 34/39] Update endpoint in rest-client --- scripts/rest_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ea3ed6edc..f9c9858ed 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -314,12 +314,12 @@ class FtRestClient(): """ return self._get(f"strategy/{strategy}") - def pairlists(self): - """Lists available pairlists + def pairlists_available(self): + """Lists available pairlist providers :return: json object """ - return self._get("pairlists") + return self._get("pairlists/available") def plot_config(self): """Return plot configuration if the strategy defines one. From 87e144a95adcee006296cfd7b16388e0c36e9d45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:24:16 +0200 Subject: [PATCH 35/39] Update webserver tags --- freqtrade/rpc/api_server/api_background_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 158a89385..e5339756b 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -76,7 +76,7 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.pairlist_running = False -@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists', 'webserver']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: @@ -125,7 +125,7 @@ def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config) @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, - tags=['pairlists']) + tags=['pairlists', 'webserver']) def pairlists_evaluate_get(jobid: str): if not (job := ApiBG.jobs.get(jobid)): raise HTTPException(status_code=404, detail='Job not found.') From fc11c79b776a386c539b62e65fb26f5445623ef0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:51:20 +0200 Subject: [PATCH 36/39] Fix not working date format output --- freqtrade/data/history/history_utils.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index dc3c7c1e6..2833a6d50 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional, Tuple from pandas import DataFrame, concat from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler @@ -227,9 +227,11 @@ def _download_pair_history(pair: str, *, ) logger.debug("Current Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("Current End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, @@ -252,10 +254,12 @@ def _download_pair_history(pair: str, *, data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair, fill_missing=False, drop_incomplete=False) - logger.debug("New Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("New End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True From 320b3e20a6f0e561659ec63ccb194fddbc3052ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 11:58:18 +0200 Subject: [PATCH 37/39] Use correct variable for candle-type when loading data closes #8757 --- freqtrade/freqai/data_drawer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b68a9dcad..55674ca18 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import Config from freqtrade.data.history import load_pair_history +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.strategy.interface import IStrategy @@ -639,7 +640,7 @@ class FreqaiDataDrawer: pair=pair, timerange=timerange, data_format=self.config.get("dataformat_ohlcv", "json"), - candle_type=self.config.get("trading_mode", "spot"), + candle_type=self.config.get("candle_type_def", CandleType.SPOT), ) def get_base_and_corr_dataframes( From 4a800fe46700f62f35b2f4876f1e04c9b34b4acb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 17:17:41 +0200 Subject: [PATCH 38/39] Add explicit test for get_stop_limit_rate --- tests/exchange/test_exchange.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f022a0905..bbf86744b 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3489,6 +3489,19 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange.stoploss_adjust(1, {}, side="sell") +@pytest.mark.parametrize('side,ratio,expected', [ + ('sell', 0.99, 99.0), # Default + ('sell', 0.999, 99.9), + ('buy', 0.99, 101.0), # Default + ('buy', 0.999, 100.1), + ]) +def test__get_stop_limit_rate(default_conf_usdt, mocker, side, ratio, expected): + exchange = get_patched_exchange(mocker, default_conf_usdt, id='binance') + + order_types = {'stoploss_on_exchange_limit_ratio': ratio} + assert exchange._get_stop_limit_rate(100, order_types, side) == expected + + def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple(EXMS, _init_ccxt=MagicMock(return_value=MagicMock()), From 5844756ba15714e4d1a0bdfed064a85d54220d42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 17:20:35 +0200 Subject: [PATCH 39/39] Add test and fix for stop-price == limit price closes #8758 --- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 88022e19c..ef3bea537 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1148,8 +1148,8 @@ class Exchange: else: limit_rate = stop_price * (2 - limit_price_pct) - bad_stop_price = ((stop_price <= limit_rate) if side == - "sell" else (stop_price >= limit_rate)) + bad_stop_price = ((stop_price < limit_rate) if side == + "sell" else (stop_price > limit_rate)) # Ensure rate is less than stop price if bad_stop_price: # This can for example happen if the stop / liquidation price is set to 0 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index bbf86744b..5fa2755d2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3492,14 +3492,22 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): @pytest.mark.parametrize('side,ratio,expected', [ ('sell', 0.99, 99.0), # Default ('sell', 0.999, 99.9), + ('sell', 1, 100), + ('sell', 1.1, InvalidOrderException), ('buy', 0.99, 101.0), # Default ('buy', 0.999, 100.1), + ('buy', 1, 100), + ('buy', 1.1, InvalidOrderException), ]) def test__get_stop_limit_rate(default_conf_usdt, mocker, side, ratio, expected): exchange = get_patched_exchange(mocker, default_conf_usdt, id='binance') order_types = {'stoploss_on_exchange_limit_ratio': ratio} - assert exchange._get_stop_limit_rate(100, order_types, side) == expected + if isinstance(expected, type) and issubclass(expected, Exception): + with pytest.raises(expected): + exchange._get_stop_limit_rate(100, order_types, side) + else: + assert exchange._get_stop_limit_rate(100, order_types, side) == expected def test_merge_ft_has_dict(default_conf, mocker):