From 2df80fc49a2b8f320a5881fbae75dc4b75077f31 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 18:35:52 +0200 Subject: [PATCH 01/70] Add /pairlists endpoint to api --- freqtrade/rpc/api_server/api_schemas.py | 4 ++++ freqtrade/rpc/api_server/api_v1.py | 26 +++++++++++++++++++------ tests/rpc/test_rpc_apiserver.py | 16 +++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 53bf7558f..64d32b15f 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -389,6 +389,10 @@ class StrategyListResponse(BaseModel): strategies: List[str] +class PairListResponse(BaseModel): + pairlists: List[Dict[str, Any]] + + class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8aa706e62..7a3a5ca84 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,11 +15,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PerformanceEntry, - Ping, PlotConfig, Profit, ResultMsg, ShowConfig, - Stats, StatusMsg, StrategyListResponse, - StrategyResponse, SysInfo, Version, - WhitelistResponse) + OpenTradeSchema, PairHistory, PairListResponse, + PerformanceEntry, Ping, PlotConfig, Profit, + ResultMsg, ShowConfig, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, SysInfo, + Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -43,7 +43,8 @@ logger = logging.getLogger(__name__) # 2.23: Allow plot config request in webserver mode # 2.24: Add cancel_open_order endpoint # 2.25: Add several profit values to /status endpoint -API_VERSION = 2.25 +# 2.26: new /pairlists endpoint +API_VERSION = 2.26 # Public API, requires no auth. router_public = APIRouter() @@ -300,6 +301,19 @@ def get_strategy(strategy: str, config=Depends(get_config)): } +@router.get('/pairlists', response_model=PairListResponse) +def list_pairlists(config=Depends(get_config)): + from freqtrade.resolvers import PairListResolver + pairlists = PairListResolver.search_all_objects( + config, False) + pairlists = sorted(pairlists, key=lambda x: x['name']) + + return {'pairlists': [{ + "name": x['name'], + } for x in pairlists + ]} + + @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 58c904838..d5a375f19 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1580,6 +1580,22 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} +def test_api_pairlists(botclient, tmpdir, mocker): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists") + + assert_response(rc) + response = rc.json() + assert isinstance(response['pairlists'], list) + assert len(response['pairlists']) > 0 + + assert len([r for r in response['pairlists'] if r['name'] == 'AgeFilter']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'VolumePairList']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'StaticPairList']) == 1 + + def test_list_available_pairs(botclient): ftbot, client = botclient From 5ad352fdf1ab1bcc5c605768f0d55f24e86d80cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 21:08:28 +0200 Subject: [PATCH 02/70] add /pairlists to rest client --- scripts/rest_client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 196542780..3c9050f43 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -313,6 +313,13 @@ class FtRestClient(): """ return self._get(f"strategy/{strategy}") + def pairlists(self): + """Lists available pairlists + + :return: json object + """ + return self._get("pairlists") + def plot_config(self): """Return plot configuration if the strategy defines one. From 987da010c97c24b52034aa0092704720aabf828e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 19 Apr 2023 21:08:44 +0200 Subject: [PATCH 03/70] Start pairlist parameter listing --- freqtrade/plugins/pairlist/AgeFilter.py | 25 ++++++++++++++++++++- freqtrade/plugins/pairlist/IPairList.py | 30 ++++++++++++++++++++++++- freqtrade/rpc/api_server/api_v1.py | 1 + 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f9c02e250..66556841b 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -12,7 +12,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import PeriodicCache @@ -68,6 +68,29 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + """ + Return parameters used by this Pairlist Handler, and their type + contains a dictionary with the parameter name as key, and a dictionary + with the type and default value. + -> Please overwrite in subclasses + """ + return { + "min_days_listed": { + "type": "number", + "default": 10, + "description": "Minimum Days Listed", + "help": "Minimum number of days a pair must have been listed on the exchange.", + }, + "max_days_listed": { + "type": "number", + "default": None, + "description": "Maximum Days Listed", + "help": "Maximum number of days a pair must have been listed on the exchange.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ :param pairlist: pairlist to filter or sort diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index d0382c778..3e143aa69 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -4,7 +4,7 @@ PairList Handler base class import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union from freqtrade.constants import Config from freqtrade.exceptions import OperationalException @@ -16,6 +16,13 @@ from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) +class PairlistParameter(TypedDict): + type: Literal["number", "string", "boolean"] + default: Union[int, float, str, bool] + description: str + help: str + + class IPairList(LoggingMixin, ABC): def __init__(self, exchange: Exchange, pairlistmanager, @@ -54,6 +61,27 @@ class IPairList(LoggingMixin, ABC): as tickers argument to filter_pairlist """ + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + """ + Return parameters used by this Pairlist Handler, and their type + contains a dictionary with the parameter name as key, and a dictionary + with the type and default value. + -> Please overwrite in subclasses + """ + return {} + + @staticmethod + def refresh_period(params: Dict[str, PairlistParameter]) -> None: + return { + "refresh_period": { + "type": "number", + "default": 1800, + "description": "Refresh period", + "help": "Refresh period in seconds", + } + } + @abstractmethod def short_desc(self) -> str: """ diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7a3a5ca84..357ccfd24 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -310,6 +310,7 @@ def list_pairlists(config=Depends(get_config)): return {'pairlists': [{ "name": x['name'], + "params": x['class'].available_parameters(), } for x in pairlists ]} From 2ea157d9d3507a3eb4f45ec806f903f9a58bc3d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 06:58:05 +0200 Subject: [PATCH 04/70] Add some more pairlist parameter definitions --- freqtrade/plugins/pairlist/AgeFilter.py | 6 ------ freqtrade/plugins/pairlist/IPairList.py | 4 ++-- freqtrade/plugins/pairlist/OffsetFilter.py | 19 ++++++++++++++++++- .../plugins/pairlist/PerformanceFilter.py | 19 ++++++++++++++++++- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- freqtrade/plugins/pairlist/StaticPairList.py | 13 ++++++++++++- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 66556841b..cd8fb9c68 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -70,12 +70,6 @@ class AgeFilter(IPairList): @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: - """ - Return parameters used by this Pairlist Handler, and their type - contains a dictionary with the parameter name as key, and a dictionary - with the type and default value. - -> Please overwrite in subclasses - """ return { "min_days_listed": { "type": "number", diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 3e143aa69..589240b10 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class PairlistParameter(TypedDict): type: Literal["number", "string", "boolean"] - default: Union[int, float, str, bool] + default: Union[int, float, str, bool, None] description: str help: str @@ -72,7 +72,7 @@ class IPairList(LoggingMixin, ABC): return {} @staticmethod - def refresh_period(params: Dict[str, PairlistParameter]) -> None: + def refresh_period_parameter() -> Dict[str, PairlistParameter]: return { "refresh_period": { "type": "number", diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 8f21cdd85..e7688a66c 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -43,6 +43,23 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "offset": { + "type": "number", + "default": 0, + "description": "Offset", + "help": "Offset of the pairlist.", + }, + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist, starting from offset.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index e7fcac1e4..6c582a4df 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -9,7 +9,7 @@ import pandas as pd from freqtrade.constants import Config from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -40,6 +40,23 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "minutes": { + "type": "number", + "default": 0, + "description": "Minutes", + "help": "Consider trades from the last X minutes. 0 means all trades.", + }, + "min_profit": { + "type": "number", + "default": None, + "description": "Minimum profit", + "help": "Minimum profit in percent. Pairs with less profit are removed.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the allowlist again. diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 2e74aa293..08f7ac5d6 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -8,7 +8,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import ROUND_UP from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 4b1961a53..f94c91d3d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -40,6 +40,17 @@ class StaticPairList(IPairList): """ return f"{self.name}" + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "allow_inactive": { + "type": "boolean", + "default": False, + "description": "Allow inactive pairs", + "help": "Allow inactive pairs to be in the whitelist.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist From 4636de30cd472321dcb8081de6f66f7e0ba568da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 07:03:27 +0200 Subject: [PATCH 05/70] Improve pairlistparam types --- freqtrade/plugins/pairlist/IPairList.py | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 589240b10..cadc706d6 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -16,13 +16,33 @@ from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) -class PairlistParameter(TypedDict): - type: Literal["number", "string", "boolean"] - default: Union[int, float, str, bool, None] +class __PairlistParameterBase(TypedDict): description: str help: str +class __NumberPairlistParameter(__PairlistParameterBase): + type: Literal["number"] + default: Union[int, float, None] + + +class __StringPairlistParameter(__PairlistParameterBase): + type: Literal["string"] + default: Union[str, None] + + +class __BoolPairlistParameter(__PairlistParameterBase): + type: Literal["boolean"] + default: Union[bool, None] + + +PairlistParameter = Union[ + __NumberPairlistParameter, + __StringPairlistParameter, + __BoolPairlistParameter + ] + + class IPairList(LoggingMixin, ABC): def __init__(self, exchange: Exchange, pairlistmanager, From e20b94d836b50bbf5930ee5cbdb3d9b0f1f60c41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 07:20:45 +0200 Subject: [PATCH 06/70] Add more filter param descriptions --- freqtrade/plugins/pairlist/PriceFilter.py | 32 +++++++++++++- .../plugins/pairlist/ProducerPairList.py | 20 ++++++++- freqtrade/plugins/pairlist/RemotePairList.py | 38 +++++++++++++++- freqtrade/plugins/pairlist/ShuffleFilter.py | 19 +++++++- freqtrade/plugins/pairlist/SpreadFilter.py | 13 +++++- .../plugins/pairlist/VolatilityFilter.py | 26 ++++++++++- freqtrade/plugins/pairlist/VolumePairList.py | 44 ++++++++++++++++++- .../plugins/pairlist/rangestabilityfilter.py | 26 ++++++++++- 8 files changed, 210 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 4d23de792..d87646f42 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -65,6 +65,36 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "low_price_ratio": { + "type": "number", + "default": 0, + "description": "Low price ratio", + "help": ("Remove pairs where a price move of 1 price unit (pip) " + "is above this ratio."), + }, + "min_price": { + "type": "number", + "default": 0, + "description": "Minimum price", + "help": "Remove pairs with a price below this value.", + }, + "max_price": { + "type": "number", + "default": 0, + "description": "Maximum price", + "help": "Remove pairs with a price above this value.", + }, + "max_value": { + "type": "number", + "default": 0, + "description": "Maximum value", + "help": "Remove pairs with a value (price * amount) above this value.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if if one price-step (pip) is > than a certain barrier. diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 882d49b76..1727efe92 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -56,6 +56,24 @@ class ProducerPairList(IPairList): """ return f"{self.name} - {self._producer_name}" + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "producer_name": { + "type": "string", + "default": "default", + "description": "Producer name", + "help": ("Name of the producer to use. Requires additional " + "external_message_consumer configuration.") + }, + } + def _filter_pairlist(self, pairlist: Optional[List[str]]): upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs( self._producer_name) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index d077330e0..a62745107 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -15,7 +15,7 @@ from freqtrade import __version__ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -63,6 +63,42 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist.", + }, + "pairlist_url": { + "type": "string", + "default": "", + "description": "URL to fetch pairlist from", + "help": "URL to fetch pairlist from", + }, + **IPairList.refresh_period_parameter(), + "keep_pairlist_on_failure": { + "type": "boolean", + "default": True, + "description": "Keep last pairlist on failure", + "help": "Keep last pairlist on failure", + }, + "read_timeout": { + "type": "number", + "default": 60, + "description": "Read timeout", + "help": "Request timeout for remote pairlist", + }, + "bearer_token": { + "type": "string", + "default": "", + "description": "Bearer token", + "help": "Bearer token - used for auth against the upstream service.", + }, + } + def process_json(self, jsonparse) -> List[str]: pairlist = jsonparse.get('pairs', []) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 76d7600d2..b566a0ac2 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -9,7 +9,7 @@ from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util.periodic_cache import PeriodicCache @@ -55,6 +55,23 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "shuffle_frequency": { + "type": "string", + "default": "candle", + "description": "Shuffle frequency", + "help": "Shuffle frequency. Can be either 'candle' or 'iteration'.", + }, + "seed": { + "type": "number", + "default": None, + "description": "Random Seed", + "help": "Seed for random number generator. Not used in live mode.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index d47b68568..9778c8072 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -45,6 +45,17 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "max_spread_ratio": { + "type": "number", + "default": 0.005, + "description": "Max spread ratio", + "help": "Max spread ratio for a pair to be considered.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Validate spread for the ticker diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 401a2e86c..59092d437 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -15,7 +15,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -63,6 +63,30 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_volatility": { + "type": "number", + "default": 0, + "description": "Minimum Volatility", + "help": "Minimum volatility a pair must have to be considered.", + }, + "max_volatility": { + "type": "number", + "default": None, + "description": "Maximum Volatility", + "help": "Maximum volatility a pair must have to be considered.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 2649a8425..4bf8ca74a 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -111,6 +111,48 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "sort_key": { + "type": "string", + "default": "quoteVolume", + "description": "Sort key", + "help": "Sort key to use for sorting the pairlist.", + }, + "min_value": { + "type": "number", + "default": 0, + "description": "Minimum value", + "help": "Minimum value to use for filtering the pairlist.", + }, + **IPairList.refresh_period_parameter(), + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "lookback_timeframe": { + "type": "string", + "default": "1d", + "description": "Lookback Timeframe", + "help": "Timeframe to use for lookback.", + }, + "lookback_period": { + "type": "number", + "default": 0, + "description": "Lookback Period", + "help": "Number of periods to look back at.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 546b026cb..fde3de1f0 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -13,7 +13,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -61,6 +61,30 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_rate_of_change": { + "type": "number", + "default": 0.01, + "description": "Minimum Rate of Change", + "help": "Minimum rate of change to filter pairs.", + }, + "max_rate_of_change": { + "type": "number", + "default": None, + "description": "Maximum Rate of Change", + "help": "Maximum rate of change to filter pairs.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range From 3ef2a57bcad765142880e904b206f1f33bc8b0db Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:09:47 +0200 Subject: [PATCH 07/70] Add "is_pairlist_generator" field to pairlists --- freqtrade/plugins/pairlist/IPairList.py | 2 ++ freqtrade/plugins/pairlist/ProducerPairList.py | 1 + freqtrade/plugins/pairlist/RemotePairList.py | 2 ++ freqtrade/plugins/pairlist/StaticPairList.py | 2 ++ freqtrade/plugins/pairlist/VolumePairList.py | 2 ++ 5 files changed, 9 insertions(+) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index cadc706d6..f752db3b1 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -45,6 +45,8 @@ PairlistParameter = Union[ class IPairList(LoggingMixin, ABC): + is_pairlist_generator = False + def __init__(self, exchange: Exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 1727efe92..623727671 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -28,6 +28,7 @@ class ProducerPairList(IPairList): } ], """ + is_pairlist_generator = True def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index a62745107..2cb157a97 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) class RemotePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index f94c91d3d..3601fb460 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -17,6 +17,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 4bf8ca74a..9bca900c7 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -25,6 +25,8 @@ SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: From 9e4f9798e628a37555089d0cf2371aa1756ca110 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:13:43 +0200 Subject: [PATCH 08/70] Add pairlist "is-generator" to api --- freqtrade/rpc/api_server/api_v1.py | 1 + tests/rpc/test_rpc_apiserver.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 357ccfd24..0b81ec6ae 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -310,6 +310,7 @@ def list_pairlists(config=Depends(get_config)): return {'pairlists': [{ "name": x['name'], + "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), } for x in pairlists ]} diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index d5a375f19..82c357fcb 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1595,6 +1595,13 @@ def test_api_pairlists(botclient, tmpdir, mocker): assert len([r for r in response['pairlists'] if r['name'] == 'VolumePairList']) == 1 assert len([r for r in response['pairlists'] if r['name'] == 'StaticPairList']) == 1 + volumepl = [r for r in response['pairlists'] if r['name'] == 'VolumePairList'][0] + assert volumepl['is_pairlist_generator'] is True + assert len(volumepl['params']) > 1 + age_pl = [r for r in response['pairlists'] if r['name'] == 'AgeFilter'][0] + assert age_pl['is_pairlist_generator'] is False + assert len(volumepl['params']) > 2 + def test_list_available_pairs(botclient): ftbot, client = botclient From c5bf029701d31b616cd425115ade8c19c2dd3a78 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 20 Apr 2023 18:15:31 +0200 Subject: [PATCH 09/70] Better type response --- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- freqtrade/rpc/api_server/api_schemas.py | 8 +++++++- freqtrade/rpc/api_server/api_v1.py | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 08f7ac5d6..2e74aa293 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -8,7 +8,7 @@ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import ROUND_UP from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 64d32b15f..13bc948d0 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -390,7 +390,13 @@ class StrategyListResponse(BaseModel): class PairListResponse(BaseModel): - pairlists: List[Dict[str, Any]] + name: str + is_pairlist_generator: bool + params: Dict[str, Any] + + +class PairListsResponse(BaseModel): + pairlists: List[PairListResponse] class FreqAIModelListResponse(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 0b81ec6ae..9eeda451e 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,7 +15,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListResponse, + OpenTradeSchema, PairHistory, PairListsResponse, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, SysInfo, @@ -301,7 +301,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListResponse) +@router.get('/pairlists', response_model=PairListsResponse) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( From 3d4be92cc6567015fa0936b070eda8edf39ac32b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 21 Apr 2023 19:30:32 +0200 Subject: [PATCH 10/70] Add option pairlist parameter type --- freqtrade/plugins/pairlist/IPairList.py | 7 +++++++ freqtrade/plugins/pairlist/ShuffleFilter.py | 3 ++- freqtrade/plugins/pairlist/VolumePairList.py | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index f752db3b1..5f1b7e591 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -31,6 +31,12 @@ class __StringPairlistParameter(__PairlistParameterBase): default: Union[str, None] +class __OptionPairlistParameter(__PairlistParameterBase): + type: Literal["option"] + default: Union[str, None] + options: List[str] + + class __BoolPairlistParameter(__PairlistParameterBase): type: Literal["boolean"] default: Union[bool, None] @@ -39,6 +45,7 @@ class __BoolPairlistParameter(__PairlistParameterBase): PairlistParameter = Union[ __NumberPairlistParameter, __StringPairlistParameter, + __OptionPairlistParameter, __BoolPairlistParameter ] diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index b566a0ac2..5419295c9 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -59,8 +59,9 @@ class ShuffleFilter(IPairList): def available_parameters() -> Dict[str, PairlistParameter]: return { "shuffle_frequency": { - "type": "string", + "type": "option", "default": "candle", + "options": ["candle", "iteration"], "description": "Shuffle frequency", "help": "Shuffle frequency. Can be either 'candle' or 'iteration'.", }, diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 9bca900c7..c6d9902a4 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,8 +123,9 @@ class VolumePairList(IPairList): "help": "Number of assets to use from the pairlist", }, "sort_key": { - "type": "string", + "type": "option", "default": "quoteVolume", + "options": SORT_VALUES, "description": "Sort key", "help": "Sort key to use for sorting the pairlist.", }, From 877d53f4399d46ee62e768982bb6f02d94615e5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 Apr 2023 20:35:24 +0200 Subject: [PATCH 11/70] Add airlists test endpoint (so pairlist configurations can be tested) --- freqtrade/rpc/api_server/api_schemas.py | 10 +++++++ freqtrade/rpc/api_server/api_v1.py | 38 +++++++++++++++++++++---- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 13bc948d0..0267ee82e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -399,6 +399,16 @@ class PairListsResponse(BaseModel): pairlists: List[PairListResponse] +class PairListsPayload(BaseModel): + pairlists: List[Dict[str, Any]] + blacklist: List[str] + stake_currency: str + + +class PairListsTest(BaseModel): + pairlists: List[PairListResponse] + + class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 9eeda451e..a8b9cfb27 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,11 +15,11 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac DeleteLockRequest, DeleteTrade, ForceEnterPayload, ForceEnterResponse, ForceExitPayload, FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListsResponse, - PerformanceEntry, Ping, PlotConfig, Profit, - ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistResponse) + OpenTradeSchema, PairHistory, PairListsPayload, + PairListsResponse, PerformanceEntry, Ping, + PlotConfig, Profit, ResultMsg, ShowConfig, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -301,7 +301,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListsResponse) +@router.get('/pairlists', response_model=PairListsResponse, tags=['pairlists', 'webserver']) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( @@ -316,6 +316,32 @@ def list_pairlists(config=Depends(get_config)): ]} +@router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) +def pairlists_test(payload: PairListsPayload, config=Depends(get_config)): + from freqtrade.plugins.pairlistmanager import PairListManager + from freqtrade.resolvers import ExchangeResolver + + config_loc = deepcopy(config) + + exchange = ExchangeResolver.load_exchange( + config_loc['exchange']['name'], config_loc, validate=False) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + + res = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + return res + + @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver From 5abd616ae998c8e92eb70d239e78d54357778a8b Mon Sep 17 00:00:00 2001 From: Achmad Fathoni Date: Tue, 2 May 2023 23:01:51 +0700 Subject: [PATCH 12/70] Fix disrepancy in freqai doc code example --- docs/freqai-configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index e7aca20be..6b4112ebd 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -43,10 +43,10 @@ The FreqAI strategy requires including the following lines of code in the standa def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # the model will return all labels created by user in `set_freqai_labels()` + # the model will return all labels created by user in `feature_engineering_*` # (& appended targets), an indication of whether or not the prediction should be accepted, # the target mean/std values for each of the labels created by user in - # `feature_engineering_*` for each training period. + # `set_freqai_targets()` for each training period. dataframe = self.freqai.start(dataframe, metadata, self) From 680e7ba98ff1d3a22b00eca4dbd224955ad63efc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:21:22 +0200 Subject: [PATCH 13/70] Get exchange through DI --- freqtrade/rpc/api_server/api_v1.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 76dea3376..3633d584b 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -329,14 +329,11 @@ def list_pairlists(config=Depends(get_config)): @router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) -def pairlists_test(payload: PairListsPayload, config=Depends(get_config)): +def pairlists_test(payload: PairListsPayload, config=Depends(get_config), exchange=Depends(get_exchange)): from freqtrade.plugins.pairlistmanager import PairListManager - from freqtrade.resolvers import ExchangeResolver config_loc = deepcopy(config) - exchange = ExchangeResolver.load_exchange( - config_loc['exchange']['name'], config_loc, validate=False) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists From 818a3342b952fbf97da446d8d9dc7387ecd12539 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:38:14 +0200 Subject: [PATCH 14/70] move pairlist evaluation to the background --- freqtrade/rpc/api_server/api_v1.py | 67 ++++++++++++++------ freqtrade/rpc/api_server/webserver_bgwork.py | 3 + 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3633d584b..c61044b61 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -2,10 +2,11 @@ import logging from copy import deepcopy from typing import List, Optional -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException from freqtrade import __version__ +from freqtrade.constants import Config from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException @@ -21,6 +22,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac StatusMsg, StrategyListResponse, StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -313,7 +315,8 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists', response_model=PairListsResponse, tags=['pairlists', 'webserver']) +@router.get('/pairlists/available', + response_model=PairListsResponse, tags=['pairlists', 'webserver']) def list_pairlists(config=Depends(get_config)): from freqtrade.resolvers import PairListResolver pairlists = PairListResolver.search_all_objects( @@ -325,30 +328,54 @@ def list_pairlists(config=Depends(get_config)): "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), } for x in pairlists - ]} + ]} -@router.post('/pairlists/test', response_model=WhitelistResponse, tags=['pairlists', 'webserver']) -def pairlists_test(payload: PairListsPayload, config=Depends(get_config), exchange=Depends(get_exchange)): - from freqtrade.plugins.pairlistmanager import PairListManager +@router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) +def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, + config=Depends(get_config)): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + def run_pairlist(config_loc: Config): + try: + from freqtrade.plugins.pairlistmanager import PairListManager + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.pairlist_result = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + + finally: + ApiBG.pairlist_running = False config_loc = deepcopy(config) - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - - res = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist + background_tasks.add_task(run_pairlist, config_loc) + ApiBG.pairlist_running = True + return { + 'status': 'Pairlist evaluation started in background.' } - return res + + +@router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) +def pairlists_evaluate_get(): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + if not ApiBG.pairlist_result: + raise HTTPException(status_code=400, detail='Pairlist not started yet.') + + return ApiBG.pairlist_result @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 925f34de3..3ab36ced1 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -14,3 +14,6 @@ class ApiBG(): bgtask_running: bool = False # Exchange - only available in webserver mode. exchange = None + # Pairlist evaluate things + pairlist_running: bool = False + pairlist_result: Dict[str, Any] = {} From 7cc8da23c21c011c3fc0d03d6c2d72dc3ed95b99 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:56:46 +0200 Subject: [PATCH 15/70] Update test for available pairlist --- tests/rpc/test_rpc_apiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index a5b816f2a..e75d2cd71 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1616,11 +1616,11 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} -def test_api_pairlists(botclient, tmpdir, mocker): +def test_api_available_pairlists(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) - rc = client_get(client, f"{BASE_URI}/pairlists") + rc = client_get(client, f"{BASE_URI}/pairlists/available") assert_response(rc) response = rc.json() From 01984a06af849124fd5dc20beb45a28c22ef254c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 09:58:38 +0200 Subject: [PATCH 16/70] Extract pairlist evaluation from sub-method --- freqtrade/rpc/api_server/api_v1.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index c61044b61..d663b17d4 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -331,36 +331,37 @@ def list_pairlists(config=Depends(get_config)): ]} +def __run_pairlist(config_loc: Config): + try: + from freqtrade.plugins.pairlistmanager import PairListManager + + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.pairlist_result = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + + finally: + ApiBG.pairlist_running = False + + @router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') - def run_pairlist(config_loc: Config): - try: - from freqtrade.plugins.pairlistmanager import PairListManager - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.pairlist_result = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - - finally: - ApiBG.pairlist_running = False - config_loc = deepcopy(config) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist - background_tasks.add_task(run_pairlist, config_loc) + background_tasks.add_task(__run_pairlist, config_loc) ApiBG.pairlist_running = True return { 'status': 'Pairlist evaluation started in background.' @@ -370,7 +371,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: raise HTTPException(status_code=400, detail='Pairlist not started yet.') From 756e1f5d5bf64d2037d4c2ea192500b9bfa8f341 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 21 May 2023 10:08:32 +0200 Subject: [PATCH 17/70] Test pairlist evaluation --- freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 54 +++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d663b17d4..52eab8754 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -374,7 +374,7 @@ def pairlists_evaluate_get(): raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: - raise HTTPException(status_code=400, detail='Pairlist not started yet.') + raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') return ApiBG.pairlist_result diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e75d2cd71..b33a093a6 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1616,7 +1616,7 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} -def test_api_available_pairlists(botclient, tmpdir): +def test_api_pairlists_available(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) @@ -1639,6 +1639,58 @@ def test_api_available_pairlists(botclient, tmpdir): assert len(volumepl['params']) > 2 +def test_api_pairlists_evaluate(botclient, tmpdir): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation not started yet.' + + ApiBG.pairlist_running = True + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' + + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC" + } + # Fail, already running + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation is already running.' + + # should start the run + ApiBG.pairlist_running = False + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc) + response = rc.json() + assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] + assert response['length'] == 4 + + # Restart with additional filter, reducing the list to 2 + body['pairlists'].append({"method": "OffsetFilter", "number_assets": 2}) + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + assert_response(rc) + response = rc.json() + assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] + assert response['length'] == 2 + + def test_list_available_pairs(botclient): ftbot, client = botclient From 33e25434b4d43c0ce9d8d3d0c03ee9bcc80e091e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 22 May 2023 19:43:27 +0200 Subject: [PATCH 18/70] Change statuscode to 202 --- freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 52eab8754..54c3a1084 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -371,7 +371,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is currently running.') + raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') if not ApiBG.pairlist_result: raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b33a093a6..6cd02f2c0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1650,7 +1650,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir): ApiBG.pairlist_running = True rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 400) + assert_response(rc, 202) assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' body = { From 4c52109fa34bfcdf86dd7ebca352c013a2b499d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 20:37:23 +0200 Subject: [PATCH 19/70] Handle pairlist evaluation errors gracefully --- freqtrade/rpc/api_server/api_v1.py | 12 ++++++++++-- freqtrade/rpc/api_server/webserver_bgwork.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 54c3a1084..411b622b4 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -4,6 +4,7 @@ from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException +from sympy import O from freqtrade import __version__ from freqtrade.constants import Config @@ -343,7 +344,9 @@ def __run_pairlist(config_loc: Config): 'length': len(pairlists.whitelist), 'whitelist': pairlists.whitelist } - + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.pairlist_error = str(e) finally: ApiBG.pairlist_running = False @@ -360,7 +363,8 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist - + ApiBG.pairlist_error = None + ApiBG.pairlist_result = {} background_tasks.add_task(__run_pairlist, config_loc) ApiBG.pairlist_running = True return { @@ -370,6 +374,10 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa @router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) def pairlists_evaluate_get(): + if ApiBG.pairlist_error: + raise HTTPException(status_code=500, + detail='Pairlist evaluation failed: ' + ApiBG.pairlist_error) + if ApiBG.pairlist_running: raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 3ab36ced1..1d2565265 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional class ApiBG(): @@ -15,5 +15,6 @@ class ApiBG(): # Exchange - only available in webserver mode. exchange = None # Pairlist evaluate things + pairlist_error: Optional[str] = None pairlist_running: bool = False pairlist_result: Dict[str, Any] = {} From 9e75c768c005beb2db351958d785b05f5d11c0d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 24 May 2023 21:01:39 +0200 Subject: [PATCH 20/70] Improve responses for evaluate get endpoints --- freqtrade/rpc/api_server/api_schemas.py | 6 ++++++ freqtrade/rpc/api_server/api_v1.py | 22 ++++++++++++++-------- tests/rpc/test_rpc_apiserver.py | 16 ++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index ae9974348..e83e343dc 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -376,6 +376,12 @@ class WhitelistResponse(BaseModel): method: List[str] +class WhitelistEvaluateResponse(BaseModel): + result: Optional[WhitelistResponse] + error: Optional[str] + status: str + + class DeleteTrade(BaseModel): cancel_order_count: int result: str diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 411b622b4..3a6618291 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -21,7 +21,8 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac PairListsResponse, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, - SysInfo, Version, WhitelistResponse) + SysInfo, Version, WhitelistEvaluateResponse, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -372,19 +373,24 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa } -@router.get('/pairlists/evaluate', response_model=WhitelistResponse, tags=['pairlists']) +@router.get('/pairlists/evaluate', response_model=WhitelistEvaluateResponse, tags=['pairlists']) def pairlists_evaluate_get(): - if ApiBG.pairlist_error: - raise HTTPException(status_code=500, - detail='Pairlist evaluation failed: ' + ApiBG.pairlist_error) if ApiBG.pairlist_running: - raise HTTPException(status_code=202, detail='Pairlist evaluation is currently running.') + return {'status': 'running'} + if ApiBG.pairlist_error: + return { + 'status': 'failed', + 'error': ApiBG.pairlist_error + } if not ApiBG.pairlist_result: - raise HTTPException(status_code=400, detail='Pairlist evaluation not started yet.') + return {'status': 'pending'} - return ApiBG.pairlist_result + return { + 'status': 'success', + 'result': ApiBG.pairlist_result + } @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6cd02f2c0..115d9ddb2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1645,13 +1645,13 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 400) - assert rc.json()['detail'] == 'Pairlist evaluation not started yet.' + assert_response(rc) + assert rc.json()['status'] == 'pending' ApiBG.pairlist_running = True rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc, 202) - assert rc.json()['detail'] == 'Pairlist evaluation is currently running.' + assert_response(rc) + assert rc.json()['status'] == 'running' body = { "pairlists": [ @@ -1675,8 +1675,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") assert_response(rc) response = rc.json() - assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] - assert response['length'] == 4 + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] + assert response['result']['length'] == 4 # Restart with additional filter, reducing the list to 2 body['pairlists'].append({"method": "OffsetFilter", "number_assets": 2}) @@ -1687,8 +1687,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") assert_response(rc) response = rc.json() - assert response['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] - assert response['length'] == 2 + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] + assert response['result']['length'] == 2 def test_list_available_pairs(botclient): From af7afa80a9546290e5012f3ff2fcbc05e3eecabd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 26 May 2023 06:44:48 +0200 Subject: [PATCH 21/70] remove gone-wrong import --- freqtrade/rpc/api_server/api_v1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3a6618291..bc78e4201 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -4,7 +4,6 @@ from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, Query from fastapi.exceptions import HTTPException -from sympy import O from freqtrade import __version__ from freqtrade.constants import Config From 1317de8c1c128c8be0c3f7e16453988a24c873b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 May 2023 18:21:23 +0200 Subject: [PATCH 22/70] Add rudimentary description per pairlist --- freqtrade/plugins/pairlist/AgeFilter.py | 4 ++++ freqtrade/plugins/pairlist/IPairList.py | 10 ++++++++++ freqtrade/plugins/pairlist/OffsetFilter.py | 4 ++++ freqtrade/plugins/pairlist/PerformanceFilter.py | 4 ++++ freqtrade/plugins/pairlist/PrecisionFilter.py | 4 ++++ freqtrade/plugins/pairlist/PriceFilter.py | 4 ++++ freqtrade/plugins/pairlist/ProducerPairList.py | 4 ++++ freqtrade/plugins/pairlist/RemotePairList.py | 4 ++++ freqtrade/plugins/pairlist/ShuffleFilter.py | 4 ++++ freqtrade/plugins/pairlist/SpreadFilter.py | 4 ++++ freqtrade/plugins/pairlist/StaticPairList.py | 4 ++++ freqtrade/plugins/pairlist/VolatilityFilter.py | 4 ++++ freqtrade/plugins/pairlist/VolumePairList.py | 4 ++++ freqtrade/plugins/pairlist/rangestabilityfilter.py | 4 ++++ freqtrade/rpc/api_server/api_schemas.py | 5 +---- freqtrade/rpc/api_server/api_v1.py | 1 + 16 files changed, 64 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 014e5fb0b..bce789446 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -68,6 +68,10 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') + @staticmethod + def description() -> str: + return "Filter pairs by age (days listed)." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 5f1b7e591..d09b447d4 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -89,6 +89,16 @@ class IPairList(LoggingMixin, ABC): If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ + return False + + @staticmethod + @abstractmethod + def description() -> str: + """ + Return description of this Pairlist Handler + -> Please overwrite in subclasses + """ + return "" @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index e7688a66c..af152c7bc 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -43,6 +43,10 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." + @staticmethod + def description() -> str: + return "Offset pair list filter." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 6c582a4df..06c504317 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -40,6 +40,10 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." + @staticmethod + def description() -> str: + return "Filter pairs by performance." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 2e74aa293..d354eaf63 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -46,6 +46,10 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." + @staticmethod + def description() -> str: + return "Filters low-value coins which would not allow setting stoplosses." + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index d87646f42..4c8781184 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -65,6 +65,10 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." + @staticmethod + def description() -> str: + return "Filter pairs by price." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 623727671..826f05913 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -57,6 +57,10 @@ class ProducerPairList(IPairList): """ return f"{self.name} - {self._producer_name}" + @staticmethod + def description() -> str: + return "Get a pairlist from an upstream bot." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 2cb157a97..372f9a593 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -65,6 +65,10 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + @staticmethod + def description() -> str: + return "Retrieve pairs from a remote API." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 5419295c9..ce37dd8b5 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -55,6 +55,10 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) + @staticmethod + def description() -> str: + return "Randomize pairlist order." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 9778c8072..ee41cbe66 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -45,6 +45,10 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") + @staticmethod + def description() -> str: + return "Filter by bid/ask difference." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 3601fb460..16fb97adb 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -42,6 +42,10 @@ class StaticPairList(IPairList): """ return f"{self.name}" + @staticmethod + def description() -> str: + return "Use pairlist as configured in config." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 6f7b9f7bc..800bf3664 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -64,6 +64,10 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filter pairs by their recent volatility." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 402092db3..e9913ad20 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -114,6 +114,10 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + @staticmethod + def description() -> str: + return "Provides dynamic pair list based on trade volumes." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index d227a284e..f294b882b 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -62,6 +62,10 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filters pairs by their rate of change." + @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index e83e343dc..d5e4b05aa 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -404,6 +404,7 @@ class StrategyListResponse(BaseModel): class PairListResponse(BaseModel): name: str + description: str is_pairlist_generator: bool params: Dict[str, Any] @@ -418,10 +419,6 @@ class PairListsPayload(BaseModel): stake_currency: str -class PairListsTest(BaseModel): - pairlists: List[PairListResponse] - - class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index bc78e4201..8e2926ea7 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -328,6 +328,7 @@ def list_pairlists(config=Depends(get_config)): "name": x['name'], "is_pairlist_generator": x['class'].is_pairlist_generator, "params": x['class'].available_parameters(), + "description": x['class'].description(), } for x in pairlists ]} From 6315516d50e901da64949d92111dc4034a8fa817 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 May 2023 15:18:46 +0200 Subject: [PATCH 23/70] Improve volumepairlist defaults --- freqtrade/plugins/pairlist/VolumePairList.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index e9913ad20..0d5e33847 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,7 +123,7 @@ class VolumePairList(IPairList): return { "number_assets": { "type": "number", - "default": 0, + "default": 30, "description": "Number of assets", "help": "Number of assets to use from the pairlist", }, @@ -143,13 +143,13 @@ class VolumePairList(IPairList): **IPairList.refresh_period_parameter(), "lookback_days": { "type": "number", - "default": 10, + "default": 0, "description": "Lookback Days", "help": "Number of days to look back at.", }, "lookback_timeframe": { "type": "string", - "default": "1d", + "default": "", "description": "Lookback Timeframe", "help": "Timeframe to use for lookback.", }, From 7bccf2129fcabd2a086c670b4dba11e2fb33b51e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 07:00:20 +0200 Subject: [PATCH 24/70] Introduce background_job endpoints --- freqtrade/rpc/api_server/api_schemas.py | 19 ++++- freqtrade/rpc/api_server/api_v1.py | 89 ++++++++++++++------ freqtrade/rpc/api_server/webserver_bgwork.py | 23 ++++- 3 files changed, 97 insertions(+), 34 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index d5e4b05aa..6ab848a50 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -27,6 +27,21 @@ class StatusMsg(BaseModel): status: str +class BgJobStarted(StatusMsg): + job_id: str + + +class BackgroundTaskStatus(BaseModel): + status: str + running: bool + progress: Optional[float] + + +class BackgroundTaskResult(BaseModel): + error: Optional[str] + status: str + + class ResultMsg(BaseModel): result: str @@ -376,10 +391,8 @@ class WhitelistResponse(BaseModel): method: List[str] -class WhitelistEvaluateResponse(BaseModel): +class WhitelistEvaluateResponse(BackgroundTaskResult): result: Optional[WhitelistResponse] - error: Optional[str] - status: str class DeleteTrade(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8e2926ea7..e9bc128ab 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -11,16 +11,17 @@ from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC -from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, - BlacklistResponse, Count, Daily, - DeleteLockRequest, DeleteTrade, ForceEnterPayload, - ForceEnterResponse, ForceExitPayload, - FreqAIModelListResponse, Health, Locks, Logs, - OpenTradeSchema, PairHistory, PairListsPayload, - PairListsResponse, PerformanceEntry, Ping, - PlotConfig, Profit, ResultMsg, ShowConfig, Stats, - StatusMsg, StrategyListResponse, StrategyResponse, - SysInfo, Version, WhitelistEvaluateResponse, +from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, BackgroundTaskStatus, Balances, + BgJobStarted, BlacklistPayload, BlacklistResponse, + Count, Daily, DeleteLockRequest, DeleteTrade, + ForceEnterPayload, ForceEnterResponse, + ForceExitPayload, FreqAIModelListResponse, Health, + Locks, Logs, OpenTradeSchema, PairHistory, + PairListsPayload, PairListsResponse, + PerformanceEntry, Ping, PlotConfig, Profit, + ResultMsg, ShowConfig, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, SysInfo, + Version, WhitelistEvaluateResponse, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional from freqtrade.rpc.api_server.webserver_bgwork import ApiBG @@ -333,26 +334,30 @@ def list_pairlists(config=Depends(get_config)): ]} -def __run_pairlist(config_loc: Config): +def __run_pairlist(job_id: str, config_loc: Config): try: + + ApiBG.jobs[job_id]['is_running'] = True from freqtrade.plugins.pairlistmanager import PairListManager exchange = get_exchange(config_loc) pairlists = PairListManager(exchange, config_loc) pairlists.refresh_pairlist() - ApiBG.pairlist_result = { + ApiBG.jobs[job_id]['result'] = { 'method': pairlists.name_list, 'length': len(pairlists.whitelist), 'whitelist': pairlists.whitelist } + ApiBG.jobs[job_id]['status'] = 'success' except (OperationalException, Exception) as e: logger.exception(e) - ApiBG.pairlist_error = str(e) + ApiBG.jobs[job_id]['error'] = str(e) finally: + ApiBG.jobs[job_id]['is_running'] = False ApiBG.pairlist_running = False -@router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists']) +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: @@ -364,32 +369,60 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist - ApiBG.pairlist_error = None - ApiBG.pairlist_result = {} - background_tasks.add_task(__run_pairlist, config_loc) + # Random job id + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + 'category': 'pairlist', + 'status': 'pending', + 'progress': None, + 'is_running': False, + 'result': {}, + 'error': None, + } + ApiBG.running_jobs.append(job_id) + background_tasks.add_task(__run_pairlist, job_id, config_loc) ApiBG.pairlist_running = True + return { - 'status': 'Pairlist evaluation started in background.' + 'status': 'Pairlist evaluation started in background.', + 'job_id': job_id, } -@router.get('/pairlists/evaluate', response_model=WhitelistEvaluateResponse, tags=['pairlists']) -def pairlists_evaluate_get(): +@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, + tags=['pairlists']) +def pairlists_evaluate_get(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') - if ApiBG.pairlist_running: - return {'status': 'running'} - if ApiBG.pairlist_error: + if job['is_running']: + raise HTTPException(status_code=400, detail='Job not finished yet.') + + if error := job['error']: return { 'status': 'failed', - 'error': ApiBG.pairlist_error + 'error': error, } - if not ApiBG.pairlist_result: - return {'status': 'pending'} - return { 'status': 'success', - 'result': ApiBG.pairlist_result + 'result': job['result'], + } + + +@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) +def background_job(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + return { + 'job_id': jobid, + # 'type': job['job_type'], + 'status': job['status'], + 'running': job['is_running'], + 'progress': job.get('progress'), + # 'job_error': job['error'], } diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 1d2565265..d4c36536c 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,5 +1,15 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Literal, Optional, TypedDict +from uuid import uuid4 + + +class JobsContainer(TypedDict): + category: Literal['pairlist'] + is_running: bool + status: str + progress: Optional[float] + result: Any + error: Optional[str] class ApiBG(): @@ -14,7 +24,14 @@ class ApiBG(): bgtask_running: bool = False # Exchange - only available in webserver mode. exchange = None + + # Generic background jobs + running_jobs: List[str] = [] + # TODO: Change this to TTLCache + jobs: Dict[str, JobsContainer] = {} # Pairlist evaluate things - pairlist_error: Optional[str] = None pairlist_running: bool = False - pairlist_result: Dict[str, Any] = {} + + @staticmethod + def get_job_id() -> str: + return str(uuid4()) From fd955028a87a125cab94deea1b1e7f6c00c60765 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 07:08:27 +0200 Subject: [PATCH 25/70] Update tests for new background method --- freqtrade/rpc/api_server/api_schemas.py | 2 ++ freqtrade/rpc/api_server/api_v1.py | 2 +- tests/rpc/test_rpc_apiserver.py | 30 ++++++++++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 6ab848a50..c3f31b3c6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -32,6 +32,8 @@ class BgJobStarted(StatusMsg): class BackgroundTaskStatus(BaseModel): + job_id: str + job_category: str status: str running: bool progress: Optional[float] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index e9bc128ab..7fb4ee248 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -418,7 +418,7 @@ def background_job(jobid: str): return { 'job_id': jobid, - # 'type': job['job_type'], + 'job_category': job['category'], 'status': job['status'], 'running': job['is_running'], 'progress': job.get('progress'), diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 115d9ddb2..476a8ff2d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1643,15 +1643,10 @@ def test_api_pairlists_evaluate(botclient, tmpdir): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") - assert_response(rc) - assert rc.json()['status'] == 'pending' - - ApiBG.pairlist_running = True - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - assert_response(rc) - assert rc.json()['status'] == 'running' + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' body = { "pairlists": [ @@ -1662,6 +1657,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir): "stake_currency": "BTC" } # Fail, already running + ApiBG.pairlist_running = True rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc, 400) assert rc.json()['detail'] == 'Pairlist evaluation is already running.' @@ -1671,8 +1667,19 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) assert rc.json()['status'] == 'Pairlist evaluation started in background.' + job_id = rc.json()['job_id'] - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + rc = client_get(client, f"{BASE_URI}/background/RandomJob") + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' + + rc = client_get(client, f"{BASE_URI}/background/{job_id}") + assert_response(rc) + response = rc.json() + assert response['job_id'] == job_id + assert response['job_category'] == 'pairlist' + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") assert_response(rc) response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] @@ -1683,8 +1690,9 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) assert rc.json()['status'] == 'Pairlist evaluation started in background.' - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") - rc = client_get(client, f"{BASE_URI}/pairlists/evaluate") + job_id = rc.json()['job_id'] + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") assert_response(rc) response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] From 88ecb935b978eb738b5ea2d656fa07345813f00c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 31 May 2023 20:22:22 +0200 Subject: [PATCH 26/70] Add "failed" state to bgjob task response --- freqtrade/rpc/api_server/api_v1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7fb4ee248..10247a1cc 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -354,6 +354,7 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.jobs[job_id]['error'] = str(e) finally: ApiBG.jobs[job_id]['is_running'] = False + ApiBG.jobs[job_id]['status'] = 'failed' ApiBG.pairlist_running = False From 77e3e9e899ea97eeacb57c0597c6838c991ec719 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Jun 2023 20:40:12 +0200 Subject: [PATCH 27/70] Move pairlists and background tasks API's to separate file --- .../rpc/api_server/api_background_tasks.py | 129 +++++++++++++++++ freqtrade/rpc/api_server/api_v1.py | 134 ++---------------- freqtrade/rpc/api_server/webserver.py | 5 + 3 files changed, 144 insertions(+), 124 deletions(-) create mode 100644 freqtrade/rpc/api_server/api_background_tasks.py diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py new file mode 100644 index 000000000..4cf2f7994 --- /dev/null +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -0,0 +1,129 @@ +import logging +from copy import deepcopy + +from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi.exceptions import HTTPException + +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, + PairListsPayload, PairListsResponse, + WhitelistEvaluateResponse) +from freqtrade.rpc.api_server.deps import get_config, get_exchange +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG + + +logger = logging.getLogger(__name__) + +# Private API, protected by authentication and webserver_mode dependency +router = APIRouter() + + +@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) +def background_job(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + return { + 'job_id': jobid, + 'job_category': job['category'], + 'status': job['status'], + 'running': job['is_running'], + 'progress': job.get('progress'), + # 'job_error': job['error'], + } + + +@router.get('/pairlists/available', + response_model=PairListsResponse, tags=['pairlists', 'webserver']) +def list_pairlists(config=Depends(get_config)): + from freqtrade.resolvers import PairListResolver + pairlists = PairListResolver.search_all_objects( + config, False) + pairlists = sorted(pairlists, key=lambda x: x['name']) + + return {'pairlists': [{ + "name": x['name'], + "is_pairlist_generator": x['class'].is_pairlist_generator, + "params": x['class'].available_parameters(), + "description": x['class'].description(), + } for x in pairlists + ]} + + +def __run_pairlist(job_id: str, config_loc: Config): + try: + + ApiBG.jobs[job_id]['is_running'] = True + from freqtrade.plugins.pairlistmanager import PairListManager + + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.jobs[job_id]['error'] = str(e) + finally: + ApiBG.jobs[job_id]['is_running'] = False + ApiBG.jobs[job_id]['status'] = 'failed' + ApiBG.pairlist_running = False + + +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) +def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, + config=Depends(get_config)): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + config_loc = deepcopy(config) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + # Random job id + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + 'category': 'pairlist', + 'status': 'pending', + 'progress': None, + 'is_running': False, + 'result': {}, + 'error': None, + } + ApiBG.running_jobs.append(job_id) + background_tasks.add_task(__run_pairlist, job_id, config_loc) + ApiBG.pairlist_running = True + + return { + 'status': 'Pairlist evaluation started in background.', + 'job_id': job_id, + } + + +@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, + tags=['pairlists']) +def pairlists_evaluate_get(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + if job['is_running']: + raise HTTPException(status_code=400, detail='Job not finished yet.') + + if error := job['error']: + return { + 'status': 'failed', + 'error': error, + } + + return { + 'status': 'success', + 'result': job['result'], + } diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 10247a1cc..c49c2b333 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -2,29 +2,25 @@ import logging from copy import deepcopy from typing import List, Optional -from fastapi import APIRouter, BackgroundTasks, Depends, Query +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException from freqtrade import __version__ -from freqtrade.constants import Config from freqtrade.data.history import get_datahandler from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC -from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, BackgroundTaskStatus, Balances, - BgJobStarted, BlacklistPayload, BlacklistResponse, - Count, Daily, DeleteLockRequest, DeleteTrade, - ForceEnterPayload, ForceEnterResponse, - ForceExitPayload, FreqAIModelListResponse, Health, - Locks, Logs, OpenTradeSchema, PairHistory, - PairListsPayload, PairListsResponse, - PerformanceEntry, Ping, PlotConfig, Profit, - ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, SysInfo, - Version, WhitelistEvaluateResponse, +from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, + BlacklistResponse, Count, Daily, + DeleteLockRequest, DeleteTrade, ForceEnterPayload, + ForceEnterResponse, ForceExitPayload, + FreqAIModelListResponse, Health, Locks, Logs, + OpenTradeSchema, PairHistory, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, ShowConfig, + Stats, StatusMsg, StrategyListResponse, + StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional -from freqtrade.rpc.api_server.webserver_bgwork import ApiBG from freqtrade.rpc.rpc import RPCException @@ -317,116 +313,6 @@ def get_strategy(strategy: str, config=Depends(get_config)): } -@router.get('/pairlists/available', - response_model=PairListsResponse, tags=['pairlists', 'webserver']) -def list_pairlists(config=Depends(get_config)): - from freqtrade.resolvers import PairListResolver - pairlists = PairListResolver.search_all_objects( - config, False) - pairlists = sorted(pairlists, key=lambda x: x['name']) - - return {'pairlists': [{ - "name": x['name'], - "is_pairlist_generator": x['class'].is_pairlist_generator, - "params": x['class'].available_parameters(), - "description": x['class'].description(), - } for x in pairlists - ]} - - -def __run_pairlist(job_id: str, config_loc: Config): - try: - - ApiBG.jobs[job_id]['is_running'] = True - from freqtrade.plugins.pairlistmanager import PairListManager - - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.jobs[job_id]['result'] = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - ApiBG.jobs[job_id]['status'] = 'success' - except (OperationalException, Exception) as e: - logger.exception(e) - ApiBG.jobs[job_id]['error'] = str(e) - finally: - ApiBG.jobs[job_id]['is_running'] = False - ApiBG.jobs[job_id]['status'] = 'failed' - ApiBG.pairlist_running = False - - -@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) -def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, - config=Depends(get_config)): - if ApiBG.pairlist_running: - raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') - - config_loc = deepcopy(config) - config_loc['stake_currency'] = payload.stake_currency - config_loc['pairlists'] = payload.pairlists - # TODO: overwrite blacklist? make it optional and fall back to the one in config? - # Outcome depends on the UI approach. - config_loc['exchange']['pair_blacklist'] = payload.blacklist - # Random job id - job_id = ApiBG.get_job_id() - - ApiBG.jobs[job_id] = { - 'category': 'pairlist', - 'status': 'pending', - 'progress': None, - 'is_running': False, - 'result': {}, - 'error': None, - } - ApiBG.running_jobs.append(job_id) - background_tasks.add_task(__run_pairlist, job_id, config_loc) - ApiBG.pairlist_running = True - - return { - 'status': 'Pairlist evaluation started in background.', - 'job_id': job_id, - } - - -@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, - tags=['pairlists']) -def pairlists_evaluate_get(jobid: str): - if not (job := ApiBG.jobs.get(jobid)): - raise HTTPException(status_code=404, detail='Job not found.') - - if job['is_running']: - raise HTTPException(status_code=400, detail='Job not finished yet.') - - if error := job['error']: - return { - 'status': 'failed', - 'error': error, - } - - return { - 'status': 'success', - 'result': job['result'], - } - - -@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) -def background_job(jobid: str): - if not (job := ApiBG.jobs.get(jobid)): - raise HTTPException(status_code=404, detail='Job not found.') - - return { - 'job_id': jobid, - 'job_category': job['category'], - 'status': job['status'], - 'running': job['is_running'], - 'progress': job.get('progress'), - # 'job_error': job['error'], - } - - @router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai']) def list_freqaimodels(config=Depends(get_config)): from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index ea623e0ed..4d934eee3 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -114,6 +114,7 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login + from freqtrade.rpc.api_server.api_background_tasks import router as api_bg_tasks from freqtrade.rpc.api_server.api_backtest import router as api_backtest from freqtrade.rpc.api_server.api_v1 import router as api_v1 from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public @@ -130,6 +131,10 @@ class ApiServer(RPCHandler): dependencies=[Depends(http_basic_or_jwt_token), Depends(is_webserver_mode)], ) + app.include_router(api_bg_tasks, prefix="/api/v1", + dependencies=[Depends(http_basic_or_jwt_token), + Depends(is_webserver_mode)], + ) app.include_router(ws_router, prefix="/api/v1") app.include_router(router_login, prefix="/api/v1", tags=["auth"]) # UI Router MUST be last! From e2594e7494418797d675a10864d49a8deebcea1d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 1 Jun 2023 20:46:28 +0200 Subject: [PATCH 28/70] Align tests to use webserver mode --- tests/rpc/test_rpc_apiserver.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9c5891b03..9d0193d4a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1622,6 +1622,12 @@ def test_api_pairlists_available(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/available") + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/available") assert_response(rc) response = rc.json() assert isinstance(response['pairlists'], list) @@ -1645,6 +1651,12 @@ def test_api_pairlists_evaluate(botclient, tmpdir): rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") assert_response(rc, 404) assert rc.json()['detail'] == 'Job not found.' From af16ce874c2bfe53b73f886bce04cf083fe68017 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Jun 2023 09:50:40 +0200 Subject: [PATCH 29/70] Allow webserver mode to cache multiple exchanges --- freqtrade/rpc/api_server/api_background_tasks.py | 1 - freqtrade/rpc/api_server/deps.py | 16 +++++++++++++--- freqtrade/rpc/api_server/webserver_bgwork.py | 8 +++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 4cf2f7994..3aa432995 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -98,7 +98,6 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa 'result': {}, 'error': None, } - ApiBG.running_jobs.append(job_id) background_tasks.add_task(__run_pairlist, job_id, config_loc) ApiBG.pairlist_running = True diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index 4c118d274..bface89bd 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -3,6 +3,7 @@ from uuid import uuid4 from fastapi import Depends, HTTPException +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.persistence import Trade from freqtrade.persistence.models import _request_id_ctx_var @@ -43,12 +44,21 @@ def get_api_config() -> Dict[str, Any]: return ApiServer._config['api_server'] +def _generate_exchange_key(config: Config) -> str: + """ + Exchange key - used for caching the exchange object. + """ + return f"{config['exchange']['name']}_{config.get('trading_mode', 'spot')}" + + def get_exchange(config=Depends(get_config)): - if not ApiBG.exchange: + exchange_key = _generate_exchange_key(config) + if not (exchange := ApiBG.exchanges.get(exchange_key)): from freqtrade.resolvers import ExchangeResolver - ApiBG.exchange = ExchangeResolver.load_exchange( + exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiBG.exchange + ApiBG.exchanges[exchange_key] = exchange + return exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index d4c36536c..3846fe138 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,7 +1,9 @@ -from typing import Any, Dict, List, Literal, Optional, TypedDict +from typing import Any, Dict, Literal, Optional, TypedDict from uuid import uuid4 +from freqtrade.exchange.exchange import Exchange + class JobsContainer(TypedDict): category: Literal['pairlist'] @@ -23,10 +25,10 @@ class ApiBG(): } bgtask_running: bool = False # Exchange - only available in webserver mode. - exchange = None + exchanges: Dict[str, Exchange] = {} # Generic background jobs - running_jobs: List[str] = [] + # TODO: Change this to TTLCache jobs: Dict[str, JobsContainer] = {} # Pairlist evaluate things From ac046d6a2da758a9cd0edfc82b80ae67005d8994 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 2 Jun 2023 10:14:11 +0200 Subject: [PATCH 30/70] Allow setting the exchange explicitly --- .../rpc/api_server/api_background_tasks.py | 4 ++++ freqtrade/rpc/api_server/api_schemas.py | 2 ++ tests/rpc/test_rpc_apiserver.py | 20 ++++++++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 3aa432995..8538f1b9a 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -84,6 +84,10 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc = deepcopy(config) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists + if payload.exchange: + config_loc['exchange']['name'] = payload.exchange + if payload.trading_mode: + config_loc['trading_mode'] = payload.trading_mode # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index c3f31b3c6..1f5a79aef 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -432,6 +432,8 @@ class PairListsPayload(BaseModel): pairlists: List[Dict[str, Any]] blacklist: List[str] stake_currency: str + trading_mode: Optional[TradingMode] + exchange: Optional[str] class FreqAIModelListResponse(BaseModel): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9d0193d4a..f4229e0a0 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1645,7 +1645,7 @@ def test_api_pairlists_available(botclient, tmpdir): assert len(volumepl['params']) > 2 -def test_api_pairlists_evaluate(botclient, tmpdir): +def test_api_pairlists_evaluate(botclient, tmpdir, mocker): ftbot, client = botclient ftbot.config['user_data_dir'] = Path(tmpdir) @@ -1709,6 +1709,24 @@ def test_api_pairlists_evaluate(botclient, tmpdir): response = rc.json() assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] assert response['result']['length'] == 2 + # Patch __run_pairlists + plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', return_value=None) + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC", + "exchange": "randomExchange", + "trading_mode": "futures", + } + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert plm.call_count == 1 + call_config = plm.call_args_list[0][0][1] + assert call_config['exchange']['name'] == 'randomExchange' + assert call_config['trading_mode'] == 'futures' def test_list_available_pairs(botclient): From 48328fb29de63bd62d0aa563ecc2e116a2d1c848 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:52:25 +0200 Subject: [PATCH 31/70] reset candle_type_def --- freqtrade/rpc/api_server/api_background_tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 8538f1b9a..8a6814801 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends from fastapi.exceptions import HTTPException from freqtrade.constants import Config +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, PairListsPayload, PairListsResponse, @@ -88,6 +89,8 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc['exchange']['name'] = payload.exchange if payload.trading_mode: config_loc['trading_mode'] = payload.trading_mode + config_loc['candle_type_def'] = CandleType.get_default( + config_loc.get('trading_mode', 'spot') or 'spot') # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist From d9d1735333d0781a2b52f0ad4b4c9f066dba7bc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:57:25 +0200 Subject: [PATCH 32/70] Extract ExchangePayload updating --- .../rpc/api_server/api_background_tasks.py | 24 ++++++++++++------- freqtrade/rpc/api_server/api_schemas.py | 9 ++++--- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 8a6814801..504ae7837 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -8,8 +8,8 @@ from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, - PairListsPayload, PairListsResponse, - WhitelistEvaluateResponse) + ExchangeModePayloadMixin, PairListsPayload, + PairListsResponse, WhitelistEvaluateResponse) from freqtrade.rpc.api_server.deps import get_config, get_exchange from freqtrade.rpc.api_server.webserver_bgwork import ApiBG @@ -85,12 +85,7 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa config_loc = deepcopy(config) config_loc['stake_currency'] = payload.stake_currency config_loc['pairlists'] = payload.pairlists - if payload.exchange: - config_loc['exchange']['name'] = payload.exchange - if payload.trading_mode: - config_loc['trading_mode'] = payload.trading_mode - config_loc['candle_type_def'] = CandleType.get_default( - config_loc.get('trading_mode', 'spot') or 'spot') + handleExchangePayload(payload, config_loc) # TODO: overwrite blacklist? make it optional and fall back to the one in config? # Outcome depends on the UI approach. config_loc['exchange']['pair_blacklist'] = payload.blacklist @@ -114,6 +109,19 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa } +def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config): + """ + Handle exchange and trading mode payload. + Updates the configuration with the payload values. + """ + if payload.exchange: + config_loc['exchange']['name'] = payload.exchange + if payload.trading_mode: + config_loc['trading_mode'] = payload.trading_mode + config_loc['candle_type_def'] = CandleType.get_default( + config_loc.get('trading_mode', 'spot') or 'spot') + + @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, tags=['pairlists']) def pairlists_evaluate_get(jobid: str): diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 1f5a79aef..b849ebeb7 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -7,6 +7,11 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode +class ExchangeModePayloadMixin(BaseModel): + trading_mode: Optional[TradingMode] + exchange: Optional[str] + + class Ping(BaseModel): status: str @@ -428,12 +433,10 @@ class PairListsResponse(BaseModel): pairlists: List[PairListResponse] -class PairListsPayload(BaseModel): +class PairListsPayload(ExchangeModePayloadMixin, BaseModel): pairlists: List[Dict[str, Any]] blacklist: List[str] stake_currency: str - trading_mode: Optional[TradingMode] - exchange: Optional[str] class FreqAIModelListResponse(BaseModel): From 10ea2b44c7753902a7a87ad43d29add7397f8868 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 3 Jun 2023 06:59:22 +0200 Subject: [PATCH 33/70] Update test line length --- tests/rpc/test_rpc_apiserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f4229e0a0..7c815d8eb 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1710,7 +1710,8 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] assert response['result']['length'] == 2 # Patch __run_pairlists - plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', return_value=None) + plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', + return_value=None) body = { "pairlists": [ {"method": "StaticPairList", }, From 71b81ee7cd9371b5696f05ba021e6d4ec42c5e82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Jun 2023 13:25:39 +0200 Subject: [PATCH 34/70] Add margin_mode to pairlists callback --- freqtrade/enums/marginmode.py | 2 +- freqtrade/rpc/api_server/api_background_tasks.py | 2 ++ freqtrade/rpc/api_server/api_schemas.py | 3 ++- tests/rpc/test_rpc_apiserver.py | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/freqtrade/enums/marginmode.py b/freqtrade/enums/marginmode.py index 1e42809ea..7fd749b29 100644 --- a/freqtrade/enums/marginmode.py +++ b/freqtrade/enums/marginmode.py @@ -1,7 +1,7 @@ from enum import Enum -class MarginMode(Enum): +class MarginMode(str, Enum): """ Enum to distinguish between cross margin/futures margin_mode and diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 504ae7837..158a89385 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -120,6 +120,8 @@ def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config) config_loc['trading_mode'] = payload.trading_mode config_loc['candle_type_def'] = CandleType.get_default( config_loc.get('trading_mode', 'spot') or 'spot') + if payload.margin_mode: + config_loc['margin_mode'] = payload.margin_mode @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 9b662b342..3f4dd99e1 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -4,12 +4,13 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf -from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode +from freqtrade.enums import MarginMode, OrderTypeValues, SignalDirection, TradingMode from freqtrade.types import ValidExchangesType class ExchangeModePayloadMixin(BaseModel): trading_mode: Optional[TradingMode] + margin_mode: Optional[MarginMode] exchange: Optional[str] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 88ba9d7e0..f793b1f9c 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1762,6 +1762,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): "stake_currency": "BTC", "exchange": "randomExchange", "trading_mode": "futures", + "margin_mode": "isolated", } rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) assert_response(rc) @@ -1769,6 +1770,7 @@ def test_api_pairlists_evaluate(botclient, tmpdir, mocker): call_config = plm.call_args_list[0][0][1] assert call_config['exchange']['name'] == 'randomExchange' assert call_config['trading_mode'] == 'futures' + assert call_config['margin_mode'] == 'isolated' def test_list_available_pairs(botclient): From f81139b97c1ce6fcb5bd000ff5b51532655e2a7c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Jun 2023 03:56:43 +0000 Subject: [PATCH 35/70] Bump python from 3.10.11-slim-bullseye to 3.11.4-slim-bullseye Bumps python from 3.10.11-slim-bullseye to 3.11.4-slim-bullseye. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3890a25b..b5f6f5d5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.11-slim-bullseye as base +FROM python:3.11.4-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From 317e0b5f2becb3b772c303556280c025f94c60ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Jun 2023 07:08:06 +0200 Subject: [PATCH 36/70] Avoid nested loops in telegram for force* scenarios closes #8731 --- freqtrade/rpc/telegram.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d082299cb..f12610d3d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1114,7 +1114,9 @@ class Telegram(RPCHandler): async def _force_exit_action(self, trade_id): if trade_id != 'cancel': try: - self._rpc._rpc_force_exit(trade_id) + loop = asyncio.get_running_loop() + # Workaround to avoid nested loops + await loop.run_in_executor(None, self._rpc._rpc_force_exit, trade_id) except RPCException as e: await self._send_msg(str(e)) @@ -1140,7 +1142,11 @@ class Telegram(RPCHandler): async def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection): if pair != 'cancel': try: - self._rpc._rpc_force_entry(pair, price, order_side=order_side) + def _force_enter(): + self._rpc._rpc_force_entry(pair, price, order_side=order_side) + loop = asyncio.get_running_loop() + # Workaround to avoid nested loops + await loop.run_in_executor(None, _force_enter) except RPCException as e: logger.exception("Forcebuy error!") await self._send_msg(str(e), ParseMode.HTML) From e3056b141ac270b38d5b22b16787813a8cf83857 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 06:51:12 +0200 Subject: [PATCH 37/70] Move logging tests to dedicated test file --- tests/test_configuration.py | 125 ---------------------------------- tests/test_log_setup.py | 130 ++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 125 deletions(-) create mode 100644 tests/test_log_setup.py diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 5b09abbd3..40aecd7c1 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,7 +1,5 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json -import logging -import sys import warnings from copy import deepcopy from pathlib import Path @@ -23,8 +21,6 @@ from freqtrade.configuration.load_config import (load_config_file, load_file, lo from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers, - setup_logging, setup_logging_pre) from tests.conftest import (CURRENT_TEST_STRATEGY, log_has, log_has_re, patched_configuration_load_config_file) @@ -605,127 +601,6 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: assert log_has('Verbosity set to 3', caplog) -def test_set_loggers() -> None: - # Reset Logging to Debug, otherwise this fails randomly as it's set globally - logging.getLogger('requests').setLevel(logging.DEBUG) - logging.getLogger("urllib3").setLevel(logging.DEBUG) - logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) - logging.getLogger('telegram').setLevel(logging.DEBUG) - - previous_value1 = logging.getLogger('requests').level - previous_value2 = logging.getLogger('ccxt.base.exchange').level - previous_value3 = logging.getLogger('telegram').level - - _set_loggers() - - value1 = logging.getLogger('requests').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('ccxt.base.exchange').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - value3 = logging.getLogger('telegram').level - assert previous_value3 is not value3 - assert value3 is logging.INFO - - _set_loggers(verbosity=2) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.INFO - assert logging.getLogger('telegram').level is logging.INFO - assert logging.getLogger('werkzeug').level is logging.INFO - - _set_loggers(verbosity=3, api_verbosity='error') - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG - assert logging.getLogger('telegram').level is logging.INFO - assert logging.getLogger('werkzeug').level is logging.ERROR - - -@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") -def test_set_loggers_syslog(): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'syslog:/dev/log', - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] - # setting up logging again should NOT cause the loggers to be added a second time. - setup_logging(config) - assert len(logger.handlers) == 3 - # reset handlers to not break pytest - logger.handlers = orig_handlers - - -@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") -def test_set_loggers_Filehandler(tmpdir): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - logfile = Path(tmpdir) / 'ft_logfile.log' - config = {'verbosity': 2, - 'logfile': str(logfile), - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] - # setting up logging again should NOT cause the loggers to be added a second time. - setup_logging(config) - assert len(logger.handlers) == 3 - # reset handlers to not break pytest - if logfile.exists: - logfile.unlink() - logger.handlers = orig_handlers - - -@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") -def test_set_loggers_journald(mocker): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'journald', - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - # reset handlers to not break pytest - logger.handlers = orig_handlers - - -def test_set_loggers_journald_importerror(import_fails): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'journald', - } - with pytest.raises(OperationalException, - match=r'You need the cysystemd python package.*'): - setup_logging(config) - logger.handlers = orig_handlers - - def test_set_logfile(default_conf, mocker, tmpdir): patched_configuration_load_config_file(mocker, default_conf) f = Path(tmpdir / "test_file.log") diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py new file mode 100644 index 000000000..276e0d27b --- /dev/null +++ b/tests/test_log_setup.py @@ -0,0 +1,130 @@ +import logging +import sys +from pathlib import Path + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers, + setup_logging, setup_logging_pre) + + +def test_set_loggers() -> None: + # Reset Logging to Debug, otherwise this fails randomly as it's set globally + logging.getLogger('requests').setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.DEBUG) + + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + _set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + _set_loggers(verbosity=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + assert logging.getLogger('werkzeug').level is logging.INFO + + _set_loggers(verbosity=3, api_verbosity='error') + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO + assert logging.getLogger('werkzeug').level is logging.ERROR + + +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +def test_set_loggers_syslog(): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'syslog:/dev/log', + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + # setting up logging again should NOT cause the loggers to be added a second time. + setup_logging(config) + assert len(logger.handlers) == 3 + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +def test_set_loggers_Filehandler(tmpdir): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + logfile = Path(tmpdir) / 'ft_logfile.log' + config = {'verbosity': 2, + 'logfile': str(logfile), + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + # setting up logging again should NOT cause the loggers to be added a second time. + setup_logging(config) + assert len(logger.handlers) == 3 + # reset handlers to not break pytest + if logfile.exists: + logfile.unlink() + logger.handlers = orig_handlers + + +@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") +def test_set_loggers_journald(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +def test_set_loggers_journald_importerror(import_fails): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + with pytest.raises(OperationalException, + match=r'You need the cysystemd python package.*'): + setup_logging(config) + logger.handlers = orig_handlers From fc8c6b06ade3074cc32e79f484311ec8d067d46e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Jun 2023 06:59:08 +0200 Subject: [PATCH 38/70] Extract set-log-levels from main logging module --- freqtrade/loggers/__init__.py | 26 ++------------------------ freqtrade/loggers/set_log_levels.py | 25 +++++++++++++++++++++++++ tests/test_configuration.py | 2 +- tests/test_log_setup.py | 8 ++++---- 4 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 freqtrade/loggers/set_log_levels.py diff --git a/freqtrade/loggers/__init__.py b/freqtrade/loggers/__init__.py index dc01e4e5b..390f210c0 100644 --- a/freqtrade/loggers/__init__.py +++ b/freqtrade/loggers/__init__.py @@ -5,6 +5,7 @@ from logging.handlers import RotatingFileHandler, SysLogHandler from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.loggers.buffering_handler import FTBufferingHandler +from freqtrade.loggers.set_log_levels import set_loggers from freqtrade.loggers.std_err_stream_handler import FTStdErrStreamHandler @@ -16,29 +17,6 @@ bufferHandler = FTBufferingHandler(1000) bufferHandler.setFormatter(Formatter(LOGFORMAT)) -def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: - """ - Set the logging level for third party libraries - :return: None - """ - - logging.getLogger('requests').setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG - ) - logging.getLogger("urllib3").setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG - ) - logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if verbosity <= 2 else logging.DEBUG - ) - logging.getLogger('telegram').setLevel(logging.INFO) - logging.getLogger('httpx').setLevel(logging.WARNING) - - logging.getLogger('werkzeug').setLevel( - logging.ERROR if api_verbosity == 'error' else logging.INFO - ) - - def get_existing_handlers(handlertype): """ Returns Existing handler or None (if the handler has not yet been added to the root handlers). @@ -115,6 +93,6 @@ def setup_logging(config: Config) -> None: logging.root.addHandler(handler_rf) logging.root.setLevel(logging.INFO if verbosity < 1 else logging.DEBUG) - _set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info')) + set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info')) logger.info('Verbosity set to %s', verbosity) diff --git a/freqtrade/loggers/set_log_levels.py b/freqtrade/loggers/set_log_levels.py new file mode 100644 index 000000000..acd8df379 --- /dev/null +++ b/freqtrade/loggers/set_log_levels.py @@ -0,0 +1,25 @@ + +import logging + + +def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: + """ + Set the logging level for third party libraries + :return: None + """ + + logging.getLogger('requests').setLevel( + logging.INFO if verbosity <= 1 else logging.DEBUG + ) + logging.getLogger("urllib3").setLevel( + logging.INFO if verbosity <= 1 else logging.DEBUG + ) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if verbosity <= 2 else logging.DEBUG + ) + logging.getLogger('telegram').setLevel(logging.INFO) + logging.getLogger('httpx').setLevel(logging.WARNING) + + logging.getLogger('werkzeug').setLevel( + logging.ERROR if api_verbosity == 'error' else logging.INFO + ) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 40aecd7c1..7808fb5c8 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -590,7 +590,7 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) # Prevent setting loggers - mocker.patch('freqtrade.loggers._set_loggers', MagicMock) + mocker.patch('freqtrade.loggers.set_loggers', MagicMock) arglist = ['trade', '-vvv'] args = Arguments(arglist).get_parsed_arg() diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py index 276e0d27b..a9be24723 100644 --- a/tests/test_log_setup.py +++ b/tests/test_log_setup.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest from freqtrade.exceptions import OperationalException -from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers, +from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, set_loggers, setup_logging, setup_logging_pre) @@ -20,7 +20,7 @@ def test_set_loggers() -> None: previous_value2 = logging.getLogger('ccxt.base.exchange').level previous_value3 = logging.getLogger('telegram').level - _set_loggers() + set_loggers() value1 = logging.getLogger('requests').level assert previous_value1 is not value1 @@ -34,14 +34,14 @@ def test_set_loggers() -> None: assert previous_value3 is not value3 assert value3 is logging.INFO - _set_loggers(verbosity=2) + set_loggers(verbosity=2) assert logging.getLogger('requests').level is logging.DEBUG assert logging.getLogger('ccxt.base.exchange').level is logging.INFO assert logging.getLogger('telegram').level is logging.INFO assert logging.getLogger('werkzeug').level is logging.INFO - _set_loggers(verbosity=3, api_verbosity='error') + set_loggers(verbosity=3, api_verbosity='error') assert logging.getLogger('requests').level is logging.DEBUG assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG From 265d782af8e906e92636fe519b0a75ca29126680 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Jun 2023 09:29:14 +0200 Subject: [PATCH 39/70] Implement the requested changes. --- docs/freqai-configuration.md | 2 +- freqtrade/templates/FreqaiExampleStrategy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 6b4112ebd..692daaf1e 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -43,7 +43,7 @@ The FreqAI strategy requires including the following lines of code in the standa def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # the model will return all labels created by user in `feature_engineering_*` + # the model will return all labels created by user in `set_freqai_targets()` # (& appended targets), an indication of whether or not the prediction should be accepted, # the target mean/std values for each of the labels created by user in # `set_freqai_targets()` for each training period. diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 493ea17f3..e0b4d045b 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -229,7 +229,7 @@ class FreqaiExampleStrategy(IStrategy): # All indicators must be populated by feature_engineering_*() functions - # the model will return all labels created by user in `feature_engineering_*` + # the model will return all labels created by user in `set_freqai_targets()` # (& appended targets), an indication of whether or not the prediction should be accepted, # the target mean/std values for each of the labels created by user in # `set_freqai_targets()` for each training period. From cfe88f06d2b990d7df7f8a1364b36ce126d6cb78 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Jun 2023 16:29:43 +0200 Subject: [PATCH 40/70] Improve behavior of okx rebuys when using stop on exchange closes #8755 --- freqtrade/exchange/okx.py | 21 +++++++++++++++++++-- tests/exchange/test_okx.py | 4 ++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index af889897c..8ad3c2cdb 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -125,6 +125,20 @@ class Okx(Exchange): params['posSide'] = self._get_posSide(side, reduceOnly) return params + def __fetch_leverage_already_set(self, pair: str, leverage: float, side: BuySell) -> bool: + try: + res_lev = self._api.fetch_leverage(symbol=pair, params={ + "mgnMode": self.margin_mode.value, + "posSide": self._get_posSide(side, False), + }) + self._log_exchange_response('get_leverage', res_lev) + already_set = all(float(x['lever']) == leverage for x in res_lev['data']) + return already_set + + except ccxt.BaseError: + # Assume all errors as "not set yet" + return False + @retrier def _lev_prep(self, pair: str, leverage: float, side: BuySell, accept_fail: bool = False): if self.trading_mode != TradingMode.SPOT and self.margin_mode is not None: @@ -141,8 +155,11 @@ class Okx(Exchange): except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e + already_set = self.__fetch_leverage_already_set(pair, leverage, side) + if not already_set: + raise TemporaryError( + f'Could not set leverage due to {e.__class__.__name__}. Message: {e}' + ) from e except ccxt.BaseError as e: raise OperationalException(e) from e diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 3824eddb7..378466ae4 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -499,7 +499,11 @@ def test__set_leverage_okx(mocker, default_conf): assert api_mock.set_leverage.call_args_list[0][1]['params'] == { 'mgnMode': 'isolated', 'posSide': 'net'} + api_mock.set_leverage = MagicMock(side_effect=ccxt.NetworkError()) + exchange._lev_prep('BTC/USDT:USDT', 3.2, 'buy') + api_mock.fetch_leverage.call_count == 1 + api_mock.fetch_leverage = MagicMock(side_effect=ccxt.NetworkError()) ccxt_exceptionhandlers( mocker, default_conf, From 2806110869e7c1f08b411fb215ade1d332e35c6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Jun 2023 16:41:37 +0200 Subject: [PATCH 41/70] Add explicit test for okx cancel_stop --- tests/exchange/test_okx.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 378466ae4..aaffff1df 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -596,3 +596,15 @@ def test_stoploss_adjust_okx(mocker, default_conf, sl1, sl2, sl3, side): } assert exchange.stoploss_adjust(sl1, order, side=side) assert not exchange.stoploss_adjust(sl2, order, side=side) + + +def test_stoploss_cancel_okx(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='okx') + + exchange.cancel_order = MagicMock() + + exchange.cancel_stoploss_order('1234', 'ETH/USDT') + assert exchange.cancel_order.call_count == 1 + assert exchange.cancel_order.call_args_list[0][1]['order_id'] == '1234' + assert exchange.cancel_order.call_args_list[0][1]['pair'] == 'ETH/USDT' + assert exchange.cancel_order.call_args_list[0][1]['params'] == {'stop': True} From e332fbfb47ba146c222f9d846d8f8d286fb416e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Jun 2023 16:55:42 +0200 Subject: [PATCH 42/70] Add explicit test for okx get_stop_params --- tests/exchange/test_okx.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index aaffff1df..e8f059118 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -608,3 +608,13 @@ def test_stoploss_cancel_okx(mocker, default_conf): assert exchange.cancel_order.call_args_list[0][1]['order_id'] == '1234' assert exchange.cancel_order.call_args_list[0][1]['pair'] == 'ETH/USDT' assert exchange.cancel_order.call_args_list[0][1]['params'] == {'stop': True} + + +def test__get_stop_params_okx(mocker, default_conf): + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + exchange = get_patched_exchange(mocker, default_conf, id='okx') + params = exchange._get_stop_params('ETH/USDT:USDT', 1500, 'sell') + + assert params['tdMode'] == 'isolated' + assert params['posSide'] == 'net' From 9ef814689e184f2425d51221107aae8d5fcac36d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:18:01 +0200 Subject: [PATCH 43/70] Update endpoint in rest-client --- scripts/rest_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ea3ed6edc..f9c9858ed 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -314,12 +314,12 @@ class FtRestClient(): """ return self._get(f"strategy/{strategy}") - def pairlists(self): - """Lists available pairlists + def pairlists_available(self): + """Lists available pairlist providers :return: json object """ - return self._get("pairlists") + return self._get("pairlists/available") def plot_config(self): """Return plot configuration if the strategy defines one. From 87e144a95adcee006296cfd7b16388e0c36e9d45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:24:16 +0200 Subject: [PATCH 44/70] Update webserver tags --- freqtrade/rpc/api_server/api_background_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index 158a89385..e5339756b 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -76,7 +76,7 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.pairlist_running = False -@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists']) +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists', 'webserver']) def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, config=Depends(get_config)): if ApiBG.pairlist_running: @@ -125,7 +125,7 @@ def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config) @router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, - tags=['pairlists']) + tags=['pairlists', 'webserver']) def pairlists_evaluate_get(jobid: str): if not (job := ApiBG.jobs.get(jobid)): raise HTTPException(status_code=404, detail='Job not found.') From fc11c79b776a386c539b62e65fb26f5445623ef0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 08:51:20 +0200 Subject: [PATCH 45/70] Fix not working date format output --- freqtrade/data/history/history_utils.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index dc3c7c1e6..2833a6d50 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional, Tuple from pandas import DataFrame, concat from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler @@ -227,9 +227,11 @@ def _download_pair_history(pair: str, *, ) logger.debug("Current Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("Current End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, @@ -252,10 +254,12 @@ def _download_pair_history(pair: str, *, data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair, fill_missing=False, drop_incomplete=False) - logger.debug("New Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("New End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True From 320b3e20a6f0e561659ec63ccb194fddbc3052ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 11:58:18 +0200 Subject: [PATCH 46/70] Use correct variable for candle-type when loading data closes #8757 --- freqtrade/freqai/data_drawer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b68a9dcad..55674ca18 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import Config from freqtrade.data.history import load_pair_history +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.strategy.interface import IStrategy @@ -639,7 +640,7 @@ class FreqaiDataDrawer: pair=pair, timerange=timerange, data_format=self.config.get("dataformat_ohlcv", "json"), - candle_type=self.config.get("trading_mode", "spot"), + candle_type=self.config.get("candle_type_def", CandleType.SPOT), ) def get_base_and_corr_dataframes( From 4a800fe46700f62f35b2f4876f1e04c9b34b4acb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 17:17:41 +0200 Subject: [PATCH 47/70] Add explicit test for get_stop_limit_rate --- tests/exchange/test_exchange.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f022a0905..bbf86744b 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3489,6 +3489,19 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange.stoploss_adjust(1, {}, side="sell") +@pytest.mark.parametrize('side,ratio,expected', [ + ('sell', 0.99, 99.0), # Default + ('sell', 0.999, 99.9), + ('buy', 0.99, 101.0), # Default + ('buy', 0.999, 100.1), + ]) +def test__get_stop_limit_rate(default_conf_usdt, mocker, side, ratio, expected): + exchange = get_patched_exchange(mocker, default_conf_usdt, id='binance') + + order_types = {'stoploss_on_exchange_limit_ratio': ratio} + assert exchange._get_stop_limit_rate(100, order_types, side) == expected + + def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple(EXMS, _init_ccxt=MagicMock(return_value=MagicMock()), From 5844756ba15714e4d1a0bdfed064a85d54220d42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Jun 2023 17:20:35 +0200 Subject: [PATCH 48/70] Add test and fix for stop-price == limit price closes #8758 --- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 88022e19c..ef3bea537 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1148,8 +1148,8 @@ class Exchange: else: limit_rate = stop_price * (2 - limit_price_pct) - bad_stop_price = ((stop_price <= limit_rate) if side == - "sell" else (stop_price >= limit_rate)) + bad_stop_price = ((stop_price < limit_rate) if side == + "sell" else (stop_price > limit_rate)) # Ensure rate is less than stop price if bad_stop_price: # This can for example happen if the stop / liquidation price is set to 0 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index bbf86744b..5fa2755d2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3492,14 +3492,22 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): @pytest.mark.parametrize('side,ratio,expected', [ ('sell', 0.99, 99.0), # Default ('sell', 0.999, 99.9), + ('sell', 1, 100), + ('sell', 1.1, InvalidOrderException), ('buy', 0.99, 101.0), # Default ('buy', 0.999, 100.1), + ('buy', 1, 100), + ('buy', 1.1, InvalidOrderException), ]) def test__get_stop_limit_rate(default_conf_usdt, mocker, side, ratio, expected): exchange = get_patched_exchange(mocker, default_conf_usdt, id='binance') order_types = {'stoploss_on_exchange_limit_ratio': ratio} - assert exchange._get_stop_limit_rate(100, order_types, side) == expected + if isinstance(expected, type) and issubclass(expected, Exception): + with pytest.raises(expected): + exchange._get_stop_limit_rate(100, order_types, side) + else: + assert exchange._get_stop_limit_rate(100, order_types, side) == expected def test_merge_ft_has_dict(default_conf, mocker): From a39f23a5c7a51d06c94291d2183c7bd49e4fa420 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:56:43 +0000 Subject: [PATCH 49/70] Bump pydantic from 1.10.8 to 1.10.9 Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.8 to 1.10.9. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.8...v1.10.9) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 118ad95e2..203a555bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ sdnotify==0.3.2 # API Server fastapi==0.96.0 -pydantic==1.10.8 +pydantic==1.10.9 uvicorn==0.22.0 pyjwt==2.7.0 aiofiles==23.1.0 From 7542909e18ca9118fa2824108598d7d19d8d4818 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:56:48 +0000 Subject: [PATCH 50/70] Bump stable-baselines3 from 2.0.0a10 to 2.0.0a13 Bumps [stable-baselines3](https://github.com/DLR-RM/stable-baselines3) from 2.0.0a10 to 2.0.0a13. - [Release notes](https://github.com/DLR-RM/stable-baselines3/releases) - [Commits](https://github.com/DLR-RM/stable-baselines3/commits) --- updated-dependencies: - dependency-name: stable-baselines3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-freqai-rl.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index de48a1da4..2672f9c38 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a10 +stable_baselines3==2.0.0a13 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 From 66dc1fd339d1e825d9d7ecbe86d3e4deca84fa7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:56:52 +0000 Subject: [PATCH 51/70] Bump filelock from 3.12.0 to 3.12.1 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.12.0 to 3.12.1. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.12.0...3.12.1) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 87b1fd3c8..20c60afe0 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,4 +5,4 @@ scipy==1.10.1 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.12.0 +filelock==3.12.1 From 71064c02e52fbbeb95a7fe393c2cc5eeeec632b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:57:05 +0000 Subject: [PATCH 52/70] Bump sqlalchemy from 2.0.15 to 2.0.16 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.15 to 2.0.16. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 118ad95e2..70f268478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==3.1.23 cryptography==41.0.1; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 -SQLAlchemy==2.0.15 +SQLAlchemy==2.0.16 python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 From a9515dee815a527b5d2229b97c50e498ecbb3dae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:57:33 +0000 Subject: [PATCH 53/70] Bump pytest from 7.3.1 to 7.3.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.1 to 7.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.1...7.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5822dd595..8461f48fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.0.270 mypy==1.3.0 pre-commit==3.3.2 -pytest==7.3.1 +pytest==7.3.2 pytest-asyncio==0.21.0 pytest-cov==4.1.0 pytest-mock==3.10.0 From 8b27b408c773e85cd2b3d56b9fbd9bdb833bc97c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:57:45 +0000 Subject: [PATCH 54/70] Bump ruff from 0.0.270 to 0.0.272 Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.270 to 0.0.272. - [Release notes](https://github.com/charliermarsh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.270...v0.0.272) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5822dd595..087b5a7a9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.270 +ruff==0.0.272 mypy==1.3.0 pre-commit==3.3.2 pytest==7.3.1 From feb6e5c4661796b8db2dc55ece2f344a6a4f8bcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:58:01 +0000 Subject: [PATCH 55/70] Bump urllib3 from 2.0.2 to 2.0.3 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.2 to 2.0.3. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.0.2...2.0.3) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 118ad95e2..a9581a719 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ httpx>=0.24.1 arrow==1.2.3 cachetools==5.3.1 requests==2.31.0 -urllib3==2.0.2 +urllib3==2.0.3 jsonschema==4.17.3 TA-Lib==0.4.26 technical==1.4.0 From 7172bc0af30f6c6f5e66a9280f53392628a8f46c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 03:58:02 +0000 Subject: [PATCH 56/70] Bump orjson from 3.9.0 to 3.9.1 Bumps [orjson](https://github.com/ijl/orjson) from 3.9.0 to 3.9.1. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.9.0...3.9.1) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 118ad95e2..644822f3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.10 # Properly format api responses -orjson==3.9.0 +orjson==3.9.1 # Notify systemd sdnotify==0.3.2 From 2e087750e055ca15541ff24723099e9ae625b650 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 05:12:52 +0000 Subject: [PATCH 57/70] Bump fastapi from 0.96.0 to 0.97.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.96.0 to 0.97.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.96.0...0.97.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 203a555bf..3f7ec1546 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ orjson==3.9.0 sdnotify==0.3.2 # API Server -fastapi==0.96.0 +fastapi==0.97.0 pydantic==1.10.9 uvicorn==0.22.0 pyjwt==2.7.0 From 21949c0446d2ce2a4ed1a8f5f36d3d56c638e01c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Jun 2023 09:23:58 +0200 Subject: [PATCH 58/70] bump sqlalchemy pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc7648cd8..476d63847 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.31.0.1 - types-tabulate==0.9.0.2 - types-python-dateutil==2.8.19.13 - - SQLAlchemy==2.0.15 + - SQLAlchemy==2.0.16 # stages: [push] - repo: https://github.com/pycqa/isort From 1beaf6f05c1802febca51895e969417466155d63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 07:26:38 +0000 Subject: [PATCH 59/70] Bump plotly from 5.14.1 to 5.15.0 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.14.1 to 5.15.0. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.14.1...v5.15.0) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 8b9ad5bc4..72303efcb 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.14.1 +plotly==5.15.0 From e763e2ad3508a03f55b7c4cc340f6104ed2ab719 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 08:08:05 +0000 Subject: [PATCH 60/70] Bump ccxt from 3.1.23 to 3.1.34 Bumps [ccxt](https://github.com/ccxt/ccxt) from 3.1.23 to 3.1.34. - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/3.1.23...3.1.34) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdf3360f8..a8c1611c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.3 pandas==2.0.2 pandas-ta==0.3.14b -ccxt==3.1.23 +ccxt==3.1.34 cryptography==41.0.1; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 From 1e44cfe2fcc7e0e4b3815eb1ba28a3e6db8ee2ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Jun 2023 07:10:52 +0200 Subject: [PATCH 61/70] Improve stoploss test --- tests/test_freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 71f494372..945a81aa6 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1241,6 +1241,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ 'type': 'stop_loss_limit', 'price': 3, 'average': 2, + 'filled': enter_order['amount'], + 'remaining': 0, 'amount': enter_order['amount'], }) mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hit) From 1a4d94a6f3cd0454fae6b7c4259ae7ee6c9c3ebd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Jun 2023 19:59:18 +0200 Subject: [PATCH 62/70] OKX stop should convert contracts to amount --- freqtrade/exchange/okx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 8ad3c2cdb..c703e3a78 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -199,6 +199,7 @@ class Okx(Exchange): order_reg['type'] = 'stoploss' order_reg['status_stop'] = 'triggered' return order_reg + order = self._order_contracts_to_amount(order) order['type'] = 'stoploss' return order From 9a7794c520163f7681058ce164a7d258c03f3695 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Jun 2023 20:29:23 +0200 Subject: [PATCH 63/70] Improve behavior for when stoploss cancels without content closes #8761 --- freqtrade/persistence/trade_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5d8aada6b..5dee2a53c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -97,7 +97,7 @@ class Order(ModelBase): @property def safe_filled(self) -> float: - return self.filled if self.filled is not None else self.amount or 0.0 + return self.filled if self.filled is not None else 0.0 @property def safe_cost(self) -> float: @@ -703,7 +703,7 @@ class LocalTrade(): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value - if self.is_open: + if self.is_open and order.safe_filled > 0: logger.info(f'{order.order_type.upper()} is hit for {self}.') else: raise ValueError(f'Unknown order type: {order.order_type}') From dec3c0f37490d25db691c92b04cb589e7a683910 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 07:02:40 +0200 Subject: [PATCH 64/70] Remove environment.yml completely --- environment.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 environment.yml diff --git a/environment.yml b/environment.yml deleted file mode 100644 index e69de29bb..000000000 From 64fcb1ed11292d419b0ded645339a180c27f66d1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 10:15:45 +0200 Subject: [PATCH 65/70] Better pin scikit-learn caused by #7896 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 106b5b6d3..b394a6877 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup plot = ['plotly>=4.0'] hyperopt = [ 'scipy', - 'scikit-learn', + 'scikit-learn<=1.1.3', 'scikit-optimize>=0.7.0', 'filelock', ] From 2107dce2cd13331953089403cde2ed491e659888 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Fri, 16 Jun 2023 15:03:49 +0200 Subject: [PATCH 66/70] Update freqai-feature-engineering.md --- docs/freqai-feature-engineering.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index 82b7569a5..d106b1dd8 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -180,6 +180,9 @@ You can ask for each of the defined features to be included also for informative In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` $= 3 * 3 * 3 * 2 * 2 = 108$. + + !!! note "Learn more about creative feature engineering" + Check out our [medium article](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) geared toward helping users learn how to creatively engineer features. ### Gain finer control over `feature_engineering_*` functions with `metadata` From ffd7394adb5429791188fa1a2e2775771e800b05 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Fri, 16 Jun 2023 15:10:11 +0200 Subject: [PATCH 67/70] Update freqai.md --- docs/freqai.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/freqai.md b/docs/freqai.md index a1b20ae1e..820fb81f6 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -107,6 +107,13 @@ This is for performance reasons - FreqAI relies on making quick predictions/retr it needs to download all the training data at the beginning of a dry/live instance. FreqAI stores and appends new candles automatically for future retrains. This means that if new pairs arrive later in the dry run due to a volume pairlist, it will not have the data ready. However, FreqAI does work with the `ShufflePairlist` or a `VolumePairlist` which keeps the total pairlist constant (but reorders the pairs according to volume). +## Additional learning materials + +Here we compile some external materials that provide deeper looks into various components of FreqAI: + +- [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495) +- [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) + ## Credits FreqAI is developed by a group of individuals who all contribute specific skillsets to the project. From 4f834c89642a4e77db1abffab93c0f76f802ea4b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 15:15:40 +0200 Subject: [PATCH 68/70] Remove old version pin for catboost --- .github/workflows/ci.yml | 1 + requirements-freqai.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ecd27cc3..0393b5cb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + check-latest: true - name: Cache_dependencies uses: actions/cache@v3 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index ad069ade2..9f6390e56 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,8 +5,7 @@ # Required for freqai scikit-learn==1.1.3 joblib==1.2.0 -catboost==1.1.1; sys_platform == 'darwin' and python_version < '3.9' -catboost==1.2; 'arm' not in platform_machine and (sys_platform != 'darwin' or python_version >= '3.9') +catboost==1.2; 'arm' not in platform_machine lightgbm==3.3.5 xgboost==1.7.5 tensorboard==2.13.0 From 7939716a5e2f5475ca18e5c8919a0bf3d3565bde Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 18:00:18 +0200 Subject: [PATCH 69/70] Improve formatting of telegram /status messages --- freqtrade/rpc/telegram.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index f12610d3d..aad7fd8c4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -534,10 +534,10 @@ class Telegram(RPCHandler): if order_nr == 1: lines.append(f"*{wording} #{order_nr}:*") lines.append( - f"*Amount:* {cur_entry_amount} " + f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})" ) - lines.append(f"*Average Price:* {cur_entry_average}") + lines.append(f"*Average Price:* {cur_entry_average:.8g}") else: sum_stake = 0 sum_amount = 0 @@ -560,9 +560,9 @@ class Telegram(RPCHandler): if is_open: lines.append("({})".format(dt_humanize(order["order_filled_date"], granularity=["day", "hour", "minute"]))) - lines.append(f"*Amount:* {cur_entry_amount} " + lines.append(f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})") - lines.append(f"*Average {wording} Price:* {cur_entry_average} " + lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} " f"({price_to_1st_entry:.2%} from 1st entry Rate)") lines.append(f"*Order filled:* {order['order_filled_date']}") @@ -633,11 +633,11 @@ class Telegram(RPCHandler): ]) lines.extend([ - "*Open Rate:* `{open_rate:.8f}`", - "*Close Rate:* `{close_rate:.8f}`" if r['close_rate'] else "", + "*Open Rate:* `{open_rate:.8g}`", + "*Close Rate:* `{close_rate:.8g}`" if r['close_rate'] else "", "*Open Date:* `{open_date}`", "*Close Date:* `{close_date}`" if r['close_date'] else "", - " \n*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", + " \n*Current Rate:* `{current_rate:.8g}`" if r['is_open'] else "", ("*Unrealized Profit:* " if r['is_open'] else "*Close Profit: *") + "`{profit_ratio:.2%}` `({profit_abs_r})`", ]) @@ -658,9 +658,9 @@ class Telegram(RPCHandler): "`({initial_stop_loss_ratio:.2%})`") # Adding stoploss and stoploss percentage only if it is not None - lines.append("*Stoploss:* `{stop_loss_abs:.8f}` " + + lines.append("*Stoploss:* `{stop_loss_abs:.8g}` " + ("`({stop_loss_ratio:.2%})`" if r['stop_loss_ratio'] else "")) - lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` " + lines.append("*Stoploss distance:* `{stoploss_current_dist:.8g}` " "`({stoploss_current_dist_ratio:.2%})`") if r['open_order']: lines.append( From 24e806f081ab6cb2b12779551534c67a630aa6f0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Jun 2023 19:58:35 +0200 Subject: [PATCH 70/70] Improve resiliance by using non-exchange controlled order attributes. --- freqtrade/strategy/interface.py | 2 +- tests/test_freqtradebot.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfdfd31d8..d0655b504 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -1300,7 +1300,7 @@ class IStrategy(ABC, HyperStrategyMixin): timedout = (order.status == 'open' and order.order_date_utc < timeout_threshold) if timedout: return True - time_method = (self.check_exit_timeout if order.side == trade.exit_side + time_method = (self.check_exit_timeout if order.ft_order_side == trade.exit_side else self.check_entry_timeout) return strategy_safe_wrapper(time_method, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 945a81aa6..24e726403 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3011,8 +3011,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) freqtrade.strategy.check_exit_timeout = MagicMock(side_effect=KeyError) freqtrade.strategy.check_entry_timeout = MagicMock(side_effect=KeyError) @@ -3020,8 +3020,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) # Return True - sells! freqtrade.strategy.check_exit_timeout = MagicMock(return_value=True) @@ -3029,8 +3029,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) trade = Trade.session.scalars(select(Trade)).first() # cancelling didn't succeed - order-id remains open. assert trade.open_order_id is not None