diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ecd27cc3..0393b5cb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + check-latest: true - name: Cache_dependencies uses: actions/cache@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc7648cd8..476d63847 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.31.0.1 - types-tabulate==0.9.0.2 - types-python-dateutil==2.8.19.13 - - SQLAlchemy==2.0.15 + - SQLAlchemy==2.0.16 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/Dockerfile b/Dockerfile index d3890a25b..b5f6f5d5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.11-slim-bullseye as base +FROM python:3.11.4-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 43c9fee75..6e2ed8379 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -43,10 +43,10 @@ The FreqAI strategy requires including the following lines of code in the standa def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # the model will return all labels created by user in `set_freqai_labels()` + # the model will return all labels created by user in `set_freqai_targets()` # (& appended targets), an indication of whether or not the prediction should be accepted, # the target mean/std values for each of the labels created by user in - # `feature_engineering_*` for each training period. + # `set_freqai_targets()` for each training period. dataframe = self.freqai.start(dataframe, metadata, self) diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index 415ac2c63..daf645339 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -180,6 +180,9 @@ You can ask for each of the defined features to be included also for informative In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` $= 3 * 3 * 3 * 2 * 2 = 108$. + + !!! note "Learn more about creative feature engineering" + Check out our [medium article](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) geared toward helping users learn how to creatively engineer features. ### Gain finer control over `feature_engineering_*` functions with `metadata` diff --git a/docs/freqai.md b/docs/freqai.md index a1b20ae1e..820fb81f6 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -107,6 +107,13 @@ This is for performance reasons - FreqAI relies on making quick predictions/retr it needs to download all the training data at the beginning of a dry/live instance. FreqAI stores and appends new candles automatically for future retrains. This means that if new pairs arrive later in the dry run due to a volume pairlist, it will not have the data ready. However, FreqAI does work with the `ShufflePairlist` or a `VolumePairlist` which keeps the total pairlist constant (but reorders the pairs according to volume). +## Additional learning materials + +Here we compile some external materials that provide deeper looks into various components of FreqAI: + +- [Real-time head-to-head: Adaptive modeling of financial market data using XGBoost and CatBoost](https://emergentmethods.medium.com/real-time-head-to-head-adaptive-modeling-of-financial-market-data-using-xgboost-and-catboost-995a115a7495) +- [FreqAI - from price to prediction](https://emergentmethods.medium.com/freqai-from-price-to-prediction-6fadac18b665) + ## Credits FreqAI is developed by a group of individuals who all contribute specific skillsets to the project. diff --git a/environment.yml b/environment.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index dc3c7c1e6..2833a6d50 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional, Tuple from pandas import DataFrame, concat from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler @@ -227,9 +227,11 @@ def _download_pair_history(pair: str, *, ) logger.debug("Current Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("Current End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') # Default since_ms to 30 days if nothing is given new_data = exchange.get_historic_ohlcv(pair=pair, @@ -252,10 +254,12 @@ def _download_pair_history(pair: str, *, data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair, fill_missing=False, drop_incomplete=False) - logger.debug("New Start: %s", - f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') logger.debug("New End: %s", - f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None') + f"{data.iloc[-1]['date']:{DATETIME_PRINT_FORMAT}}" + if not data.empty else 'None') data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True diff --git a/freqtrade/enums/marginmode.py b/freqtrade/enums/marginmode.py index 1e42809ea..7fd749b29 100644 --- a/freqtrade/enums/marginmode.py +++ b/freqtrade/enums/marginmode.py @@ -1,7 +1,7 @@ from enum import Enum -class MarginMode(Enum): +class MarginMode(str, Enum): """ Enum to distinguish between cross margin/futures margin_mode and diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 88022e19c..ef3bea537 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1148,8 +1148,8 @@ class Exchange: else: limit_rate = stop_price * (2 - limit_price_pct) - bad_stop_price = ((stop_price <= limit_rate) if side == - "sell" else (stop_price >= limit_rate)) + bad_stop_price = ((stop_price < limit_rate) if side == + "sell" else (stop_price > limit_rate)) # Ensure rate is less than stop price if bad_stop_price: # This can for example happen if the stop / liquidation price is set to 0 diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index af889897c..c703e3a78 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -125,6 +125,20 @@ class Okx(Exchange): params['posSide'] = self._get_posSide(side, reduceOnly) return params + def __fetch_leverage_already_set(self, pair: str, leverage: float, side: BuySell) -> bool: + try: + res_lev = self._api.fetch_leverage(symbol=pair, params={ + "mgnMode": self.margin_mode.value, + "posSide": self._get_posSide(side, False), + }) + self._log_exchange_response('get_leverage', res_lev) + already_set = all(float(x['lever']) == leverage for x in res_lev['data']) + return already_set + + except ccxt.BaseError: + # Assume all errors as "not set yet" + return False + @retrier def _lev_prep(self, pair: str, leverage: float, side: BuySell, accept_fail: bool = False): if self.trading_mode != TradingMode.SPOT and self.margin_mode is not None: @@ -141,8 +155,11 @@ class Okx(Exchange): except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e + already_set = self.__fetch_leverage_already_set(pair, leverage, side) + if not already_set: + raise TemporaryError( + f'Could not set leverage due to {e.__class__.__name__}. Message: {e}' + ) from e except ccxt.BaseError as e: raise OperationalException(e) from e @@ -182,6 +199,7 @@ class Okx(Exchange): order_reg['type'] = 'stoploss' order_reg['status_stop'] = 'triggered' return order_reg + order = self._order_contracts_to_amount(order) order['type'] = 'stoploss' return order diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index edd9640c9..5fdb8be25 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import Config from freqtrade.data.history import load_pair_history +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.strategy.interface import IStrategy @@ -635,7 +636,7 @@ class FreqaiDataDrawer: pair=pair, timerange=timerange, data_format=self.config.get("dataformat_ohlcv", "json"), - candle_type=self.config.get("trading_mode", "spot"), + candle_type=self.config.get("candle_type_def", CandleType.SPOT), ) def get_base_and_corr_dataframes( diff --git a/freqtrade/loggers/__init__.py b/freqtrade/loggers/__init__.py index dc01e4e5b..390f210c0 100644 --- a/freqtrade/loggers/__init__.py +++ b/freqtrade/loggers/__init__.py @@ -5,6 +5,7 @@ from logging.handlers import RotatingFileHandler, SysLogHandler from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.loggers.buffering_handler import FTBufferingHandler +from freqtrade.loggers.set_log_levels import set_loggers from freqtrade.loggers.std_err_stream_handler import FTStdErrStreamHandler @@ -16,29 +17,6 @@ bufferHandler = FTBufferingHandler(1000) bufferHandler.setFormatter(Formatter(LOGFORMAT)) -def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: - """ - Set the logging level for third party libraries - :return: None - """ - - logging.getLogger('requests').setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG - ) - logging.getLogger("urllib3").setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG - ) - logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if verbosity <= 2 else logging.DEBUG - ) - logging.getLogger('telegram').setLevel(logging.INFO) - logging.getLogger('httpx').setLevel(logging.WARNING) - - logging.getLogger('werkzeug').setLevel( - logging.ERROR if api_verbosity == 'error' else logging.INFO - ) - - def get_existing_handlers(handlertype): """ Returns Existing handler or None (if the handler has not yet been added to the root handlers). @@ -115,6 +93,6 @@ def setup_logging(config: Config) -> None: logging.root.addHandler(handler_rf) logging.root.setLevel(logging.INFO if verbosity < 1 else logging.DEBUG) - _set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info')) + set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info')) logger.info('Verbosity set to %s', verbosity) diff --git a/freqtrade/loggers/set_log_levels.py b/freqtrade/loggers/set_log_levels.py new file mode 100644 index 000000000..acd8df379 --- /dev/null +++ b/freqtrade/loggers/set_log_levels.py @@ -0,0 +1,25 @@ + +import logging + + +def set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: + """ + Set the logging level for third party libraries + :return: None + """ + + logging.getLogger('requests').setLevel( + logging.INFO if verbosity <= 1 else logging.DEBUG + ) + logging.getLogger("urllib3").setLevel( + logging.INFO if verbosity <= 1 else logging.DEBUG + ) + logging.getLogger('ccxt.base.exchange').setLevel( + logging.INFO if verbosity <= 2 else logging.DEBUG + ) + logging.getLogger('telegram').setLevel(logging.INFO) + logging.getLogger('httpx').setLevel(logging.WARNING) + + logging.getLogger('werkzeug').setLevel( + logging.ERROR if api_verbosity == 'error' else logging.INFO + ) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5d8aada6b..5dee2a53c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -97,7 +97,7 @@ class Order(ModelBase): @property def safe_filled(self) -> float: - return self.filled if self.filled is not None else self.amount or 0.0 + return self.filled if self.filled is not None else 0.0 @property def safe_cost(self) -> float: @@ -703,7 +703,7 @@ class LocalTrade(): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss self.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value - if self.is_open: + if self.is_open and order.safe_filled > 0: logger.info(f'{order.order_type.upper()} is hit for {self}.') else: raise ValueError(f'Unknown order type: {order.order_type}') diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 2af86592f..bce789446 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -12,7 +12,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import PeriodicCache, dt_floor_day, dt_now, dt_ts @@ -68,6 +68,27 @@ class AgeFilter(IPairList): f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" ) if self._max_days_listed else '') + @staticmethod + def description() -> str: + return "Filter pairs by age (days listed)." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "min_days_listed": { + "type": "number", + "default": 10, + "description": "Minimum Days Listed", + "help": "Minimum number of days a pair must have been listed on the exchange.", + }, + "max_days_listed": { + "type": "number", + "default": None, + "description": "Maximum Days Listed", + "help": "Maximum number of days a pair must have been listed on the exchange.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ :param pairlist: pairlist to filter or sort diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index d0382c778..d09b447d4 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -4,7 +4,7 @@ PairList Handler base class import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union from freqtrade.constants import Config from freqtrade.exceptions import OperationalException @@ -16,8 +16,44 @@ from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) +class __PairlistParameterBase(TypedDict): + description: str + help: str + + +class __NumberPairlistParameter(__PairlistParameterBase): + type: Literal["number"] + default: Union[int, float, None] + + +class __StringPairlistParameter(__PairlistParameterBase): + type: Literal["string"] + default: Union[str, None] + + +class __OptionPairlistParameter(__PairlistParameterBase): + type: Literal["option"] + default: Union[str, None] + options: List[str] + + +class __BoolPairlistParameter(__PairlistParameterBase): + type: Literal["boolean"] + default: Union[bool, None] + + +PairlistParameter = Union[ + __NumberPairlistParameter, + __StringPairlistParameter, + __OptionPairlistParameter, + __BoolPairlistParameter + ] + + class IPairList(LoggingMixin, ABC): + is_pairlist_generator = False + def __init__(self, exchange: Exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: @@ -53,6 +89,37 @@ class IPairList(LoggingMixin, ABC): If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ + return False + + @staticmethod + @abstractmethod + def description() -> str: + """ + Return description of this Pairlist Handler + -> Please overwrite in subclasses + """ + return "" + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + """ + Return parameters used by this Pairlist Handler, and their type + contains a dictionary with the parameter name as key, and a dictionary + with the type and default value. + -> Please overwrite in subclasses + """ + return {} + + @staticmethod + def refresh_period_parameter() -> Dict[str, PairlistParameter]: + return { + "refresh_period": { + "type": "number", + "default": 1800, + "description": "Refresh period", + "help": "Refresh period in seconds", + } + } @abstractmethod def short_desc(self) -> str: diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 8f21cdd85..af152c7bc 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -43,6 +43,27 @@ class OffsetFilter(IPairList): return f"{self.name} - Taking {self._number_pairs} Pairs, starting from {self._offset}." return f"{self.name} - Offsetting pairs by {self._offset}." + @staticmethod + def description() -> str: + return "Offset pair list filter." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "offset": { + "type": "number", + "default": 0, + "description": "Offset", + "help": "Offset of the pairlist.", + }, + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist, starting from offset.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index e7fcac1e4..06c504317 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -9,7 +9,7 @@ import pandas as pd from freqtrade.constants import Config from freqtrade.exchange.types import Tickers from freqtrade.persistence import Trade -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -40,6 +40,27 @@ class PerformanceFilter(IPairList): """ return f"{self.name} - Sorting pairs by performance." + @staticmethod + def description() -> str: + return "Filter pairs by performance." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "minutes": { + "type": "number", + "default": 0, + "description": "Minutes", + "help": "Consider trades from the last X minutes. 0 means all trades.", + }, + "min_profit": { + "type": "number", + "default": None, + "description": "Minimum profit", + "help": "Minimum profit in percent. Pairs with less profit are removed.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the allowlist again. diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 2e74aa293..d354eaf63 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -46,6 +46,10 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." + @staticmethod + def description() -> str: + return "Filters low-value coins which would not allow setting stoplosses." + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 4d23de792..4c8781184 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -65,6 +65,40 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." + @staticmethod + def description() -> str: + return "Filter pairs by price." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "low_price_ratio": { + "type": "number", + "default": 0, + "description": "Low price ratio", + "help": ("Remove pairs where a price move of 1 price unit (pip) " + "is above this ratio."), + }, + "min_price": { + "type": "number", + "default": 0, + "description": "Minimum price", + "help": "Remove pairs with a price below this value.", + }, + "max_price": { + "type": "number", + "default": 0, + "description": "Maximum price", + "help": "Remove pairs with a price above this value.", + }, + "max_value": { + "type": "number", + "default": 0, + "description": "Maximum value", + "help": "Remove pairs with a value (price * amount) above this value.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Check if if one price-step (pip) is > than a certain barrier. diff --git a/freqtrade/plugins/pairlist/ProducerPairList.py b/freqtrade/plugins/pairlist/ProducerPairList.py index 882d49b76..826f05913 100644 --- a/freqtrade/plugins/pairlist/ProducerPairList.py +++ b/freqtrade/plugins/pairlist/ProducerPairList.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -28,6 +28,7 @@ class ProducerPairList(IPairList): } ], """ + is_pairlist_generator = True def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], @@ -56,6 +57,28 @@ class ProducerPairList(IPairList): """ return f"{self.name} - {self._producer_name}" + @staticmethod + def description() -> str: + return "Get a pairlist from an upstream bot." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "producer_name": { + "type": "string", + "default": "default", + "description": "Producer name", + "help": ("Name of the producer to use. Requires additional " + "external_message_consumer configuration.") + }, + } + def _filter_pairlist(self, pairlist: Optional[List[str]]): upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs( self._producer_name) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index d077330e0..372f9a593 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -15,7 +15,7 @@ from freqtrade import __version__ from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) class RemotePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: @@ -63,6 +65,46 @@ class RemotePairList(IPairList): """ return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist." + @staticmethod + def description() -> str: + return "Retrieve pairs from a remote API." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 0, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist.", + }, + "pairlist_url": { + "type": "string", + "default": "", + "description": "URL to fetch pairlist from", + "help": "URL to fetch pairlist from", + }, + **IPairList.refresh_period_parameter(), + "keep_pairlist_on_failure": { + "type": "boolean", + "default": True, + "description": "Keep last pairlist on failure", + "help": "Keep last pairlist on failure", + }, + "read_timeout": { + "type": "number", + "default": 60, + "description": "Read timeout", + "help": "Request timeout for remote pairlist", + }, + "bearer_token": { + "type": "string", + "default": "", + "description": "Bearer token", + "help": "Bearer token - used for auth against the upstream service.", + }, + } + def process_json(self, jsonparse) -> List[str]: pairlist = jsonparse.get('pairs', []) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 76d7600d2..ce37dd8b5 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -9,7 +9,7 @@ from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util.periodic_cache import PeriodicCache @@ -55,6 +55,28 @@ class ShuffleFilter(IPairList): return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" + (f", seed = {self._seed}." if self._seed is not None else ".")) + @staticmethod + def description() -> str: + return "Randomize pairlist order." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "shuffle_frequency": { + "type": "option", + "default": "candle", + "options": ["candle", "iteration"], + "description": "Shuffle frequency", + "help": "Shuffle frequency. Can be either 'candle' or 'iteration'.", + }, + "seed": { + "type": "number", + "default": None, + "description": "Random Seed", + "help": "Seed for random number generator. Not used in live mode.", + }, + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index d47b68568..ee41cbe66 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Optional from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -45,6 +45,21 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio:.2%}.") + @staticmethod + def description() -> str: + return "Filter by bid/ask difference." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "max_spread_ratio": { + "type": "number", + "default": 0.005, + "description": "Max spread ratio", + "help": "Max spread ratio for a pair to be considered.", + }, + } + def _validate_pair(self, pair: str, ticker: Optional[Ticker]) -> bool: """ Validate spread for the ticker diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 4b1961a53..16fb97adb 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List from freqtrade.constants import Config from freqtrade.exchange.types import Tickers -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) @@ -17,6 +17,8 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: @@ -40,6 +42,21 @@ class StaticPairList(IPairList): """ return f"{self.name}" + @staticmethod + def description() -> str: + return "Use pairlist as configured in config." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "allow_inactive": { + "type": "boolean", + "default": False, + "description": "Allow inactive pairs", + "help": "Allow inactive pairs to be in the whitelist.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 61a1dcbf0..800bf3664 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -15,7 +15,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import dt_floor_day, dt_now, dt_ts @@ -64,6 +64,34 @@ class VolatilityFilter(IPairList): f"{self._min_volatility}-{self._max_volatility} " f" the last {self._days} {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filter pairs by their recent volatility." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_volatility": { + "type": "number", + "default": 0, + "description": "Minimum Volatility", + "help": "Minimum volatility a pair must have to be considered.", + }, + "max_volatility": { + "type": "number", + "default": None, + "description": "Maximum Volatility", + "help": "Maximum volatility a pair must have to be considered.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index b9c312f87..0d5e33847 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -14,7 +14,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.misc import format_ms_time -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import dt_now @@ -26,6 +26,8 @@ SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): + is_pairlist_generator = True + def __init__(self, exchange, pairlistmanager, config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: @@ -112,6 +114,53 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + @staticmethod + def description() -> str: + return "Provides dynamic pair list based on trade volumes." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "number_assets": { + "type": "number", + "default": 30, + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", + }, + "sort_key": { + "type": "option", + "default": "quoteVolume", + "options": SORT_VALUES, + "description": "Sort key", + "help": "Sort key to use for sorting the pairlist.", + }, + "min_value": { + "type": "number", + "default": 0, + "description": "Minimum value", + "help": "Minimum value to use for filtering the pairlist.", + }, + **IPairList.refresh_period_parameter(), + "lookback_days": { + "type": "number", + "default": 0, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "lookback_timeframe": { + "type": "string", + "default": "", + "description": "Lookback Timeframe", + "help": "Timeframe to use for lookback.", + }, + "lookback_period": { + "type": "number", + "default": 0, + "description": "Lookback Period", + "help": "Number of periods to look back at.", + }, + } + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 1181b2812..f294b882b 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -13,7 +13,7 @@ from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.misc import plural -from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import dt_floor_day, dt_now, dt_ts @@ -62,6 +62,34 @@ class RangeStabilityFilter(IPairList): f"{self._min_rate_of_change}{max_rate_desc} over the " f"last {plural(self._days, 'day')}.") + @staticmethod + def description() -> str: + return "Filters pairs by their rate of change." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "lookback_days": { + "type": "number", + "default": 10, + "description": "Lookback Days", + "help": "Number of days to look back at.", + }, + "min_rate_of_change": { + "type": "number", + "default": 0.01, + "description": "Minimum Rate of Change", + "help": "Minimum rate of change to filter pairs.", + }, + "max_rate_of_change": { + "type": "number", + "default": None, + "description": "Maximum Rate of Change", + "help": "Maximum rate of change to filter pairs.", + }, + **IPairList.refresh_period_parameter() + } + def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]: """ Validate trading range diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py new file mode 100644 index 000000000..e5339756b --- /dev/null +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -0,0 +1,145 @@ +import logging +from copy import deepcopy + +from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi.exceptions import HTTPException + +from freqtrade.constants import Config +from freqtrade.enums import CandleType +from freqtrade.exceptions import OperationalException +from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, + ExchangeModePayloadMixin, PairListsPayload, + PairListsResponse, WhitelistEvaluateResponse) +from freqtrade.rpc.api_server.deps import get_config, get_exchange +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG + + +logger = logging.getLogger(__name__) + +# Private API, protected by authentication and webserver_mode dependency +router = APIRouter() + + +@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver']) +def background_job(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + return { + 'job_id': jobid, + 'job_category': job['category'], + 'status': job['status'], + 'running': job['is_running'], + 'progress': job.get('progress'), + # 'job_error': job['error'], + } + + +@router.get('/pairlists/available', + response_model=PairListsResponse, tags=['pairlists', 'webserver']) +def list_pairlists(config=Depends(get_config)): + from freqtrade.resolvers import PairListResolver + pairlists = PairListResolver.search_all_objects( + config, False) + pairlists = sorted(pairlists, key=lambda x: x['name']) + + return {'pairlists': [{ + "name": x['name'], + "is_pairlist_generator": x['class'].is_pairlist_generator, + "params": x['class'].available_parameters(), + "description": x['class'].description(), + } for x in pairlists + ]} + + +def __run_pairlist(job_id: str, config_loc: Config): + try: + + ApiBG.jobs[job_id]['is_running'] = True + from freqtrade.plugins.pairlistmanager import PairListManager + + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.jobs[job_id]['error'] = str(e) + finally: + ApiBG.jobs[job_id]['is_running'] = False + ApiBG.jobs[job_id]['status'] = 'failed' + ApiBG.pairlist_running = False + + +@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists', 'webserver']) +def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks, + config=Depends(get_config)): + if ApiBG.pairlist_running: + raise HTTPException(status_code=400, detail='Pairlist evaluation is already running.') + + config_loc = deepcopy(config) + config_loc['stake_currency'] = payload.stake_currency + config_loc['pairlists'] = payload.pairlists + handleExchangePayload(payload, config_loc) + # TODO: overwrite blacklist? make it optional and fall back to the one in config? + # Outcome depends on the UI approach. + config_loc['exchange']['pair_blacklist'] = payload.blacklist + # Random job id + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + 'category': 'pairlist', + 'status': 'pending', + 'progress': None, + 'is_running': False, + 'result': {}, + 'error': None, + } + background_tasks.add_task(__run_pairlist, job_id, config_loc) + ApiBG.pairlist_running = True + + return { + 'status': 'Pairlist evaluation started in background.', + 'job_id': job_id, + } + + +def handleExchangePayload(payload: ExchangeModePayloadMixin, config_loc: Config): + """ + Handle exchange and trading mode payload. + Updates the configuration with the payload values. + """ + if payload.exchange: + config_loc['exchange']['name'] = payload.exchange + if payload.trading_mode: + config_loc['trading_mode'] = payload.trading_mode + config_loc['candle_type_def'] = CandleType.get_default( + config_loc.get('trading_mode', 'spot') or 'spot') + if payload.margin_mode: + config_loc['margin_mode'] = payload.margin_mode + + +@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse, + tags=['pairlists', 'webserver']) +def pairlists_evaluate_get(jobid: str): + if not (job := ApiBG.jobs.get(jobid)): + raise HTTPException(status_code=404, detail='Job not found.') + + if job['is_running']: + raise HTTPException(status_code=400, detail='Job not finished yet.') + + if error := job['error']: + return { + 'status': 'failed', + 'error': error, + } + + return { + 'status': 'success', + 'result': job['result'], + } diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index e218465fc..3f4dd99e1 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -4,10 +4,16 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel from freqtrade.constants import DATETIME_PRINT_FORMAT, IntOrInf -from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode +from freqtrade.enums import MarginMode, OrderTypeValues, SignalDirection, TradingMode from freqtrade.types import ValidExchangesType +class ExchangeModePayloadMixin(BaseModel): + trading_mode: Optional[TradingMode] + margin_mode: Optional[MarginMode] + exchange: Optional[str] + + class Ping(BaseModel): status: str @@ -28,6 +34,23 @@ class StatusMsg(BaseModel): status: str +class BgJobStarted(StatusMsg): + job_id: str + + +class BackgroundTaskStatus(BaseModel): + job_id: str + job_category: str + status: str + running: bool + progress: Optional[float] + + +class BackgroundTaskResult(BaseModel): + error: Optional[str] + status: str + + class ResultMsg(BaseModel): result: str @@ -377,6 +400,10 @@ class WhitelistResponse(BaseModel): method: List[str] +class WhitelistEvaluateResponse(BackgroundTaskResult): + result: Optional[WhitelistResponse] + + class DeleteTrade(BaseModel): cancel_order_count: int result: str @@ -401,6 +428,23 @@ class ExchangeListResponse(BaseModel): exchanges: List[ValidExchangesType] +class PairListResponse(BaseModel): + name: str + description: str + is_pairlist_generator: bool + params: Dict[str, Any] + + +class PairListsResponse(BaseModel): + pairlists: List[PairListResponse] + + +class PairListsPayload(ExchangeModePayloadMixin, BaseModel): + pairlists: List[Dict[str, Any]] + blacklist: List[str] + stake_currency: str + + class FreqAIModelListResponse(BaseModel): freqaimodels: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 6c0ea04aa..143f110f0 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -48,7 +48,8 @@ logger = logging.getLogger(__name__) # 2.27: Add /trades//reload endpoint # 2.28: Switch reload endpoint to Post # 2.29: Add /exchanges endpoint -API_VERSION = 2.29 +# 2.30: new /pairlists endpoint +API_VERSION = 2.30 # Public API, requires no auth. router_public = APIRouter() diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index 4c118d274..bface89bd 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -3,6 +3,7 @@ from uuid import uuid4 from fastapi import Depends, HTTPException +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.persistence import Trade from freqtrade.persistence.models import _request_id_ctx_var @@ -43,12 +44,21 @@ def get_api_config() -> Dict[str, Any]: return ApiServer._config['api_server'] +def _generate_exchange_key(config: Config) -> str: + """ + Exchange key - used for caching the exchange object. + """ + return f"{config['exchange']['name']}_{config.get('trading_mode', 'spot')}" + + def get_exchange(config=Depends(get_config)): - if not ApiBG.exchange: + exchange_key = _generate_exchange_key(config) + if not (exchange := ApiBG.exchanges.get(exchange_key)): from freqtrade.resolvers import ExchangeResolver - ApiBG.exchange = ExchangeResolver.load_exchange( + exchange = ExchangeResolver.load_exchange( config, load_leverage_tiers=False) - return ApiBG.exchange + ApiBG.exchanges[exchange_key] = exchange + return exchange def get_message_stream(): diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index ea623e0ed..4d934eee3 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -114,6 +114,7 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login + from freqtrade.rpc.api_server.api_background_tasks import router as api_bg_tasks from freqtrade.rpc.api_server.api_backtest import router as api_backtest from freqtrade.rpc.api_server.api_v1 import router as api_v1 from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public @@ -130,6 +131,10 @@ class ApiServer(RPCHandler): dependencies=[Depends(http_basic_or_jwt_token), Depends(is_webserver_mode)], ) + app.include_router(api_bg_tasks, prefix="/api/v1", + dependencies=[Depends(http_basic_or_jwt_token), + Depends(is_webserver_mode)], + ) app.include_router(ws_router, prefix="/api/v1") app.include_router(router_login, prefix="/api/v1", tags=["auth"]) # UI Router MUST be last! diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 925f34de3..3846fe138 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -1,5 +1,17 @@ -from typing import Any, Dict +from typing import Any, Dict, Literal, Optional, TypedDict +from uuid import uuid4 + +from freqtrade.exchange.exchange import Exchange + + +class JobsContainer(TypedDict): + category: Literal['pairlist'] + is_running: bool + status: str + progress: Optional[float] + result: Any + error: Optional[str] class ApiBG(): @@ -13,4 +25,15 @@ class ApiBG(): } bgtask_running: bool = False # Exchange - only available in webserver mode. - exchange = None + exchanges: Dict[str, Exchange] = {} + + # Generic background jobs + + # TODO: Change this to TTLCache + jobs: Dict[str, JobsContainer] = {} + # Pairlist evaluate things + pairlist_running: bool = False + + @staticmethod + def get_job_id() -> str: + return str(uuid4()) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d082299cb..aad7fd8c4 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -534,10 +534,10 @@ class Telegram(RPCHandler): if order_nr == 1: lines.append(f"*{wording} #{order_nr}:*") lines.append( - f"*Amount:* {cur_entry_amount} " + f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})" ) - lines.append(f"*Average Price:* {cur_entry_average}") + lines.append(f"*Average Price:* {cur_entry_average:.8g}") else: sum_stake = 0 sum_amount = 0 @@ -560,9 +560,9 @@ class Telegram(RPCHandler): if is_open: lines.append("({})".format(dt_humanize(order["order_filled_date"], granularity=["day", "hour", "minute"]))) - lines.append(f"*Amount:* {cur_entry_amount} " + lines.append(f"*Amount:* {cur_entry_amount:.8g} " f"({round_coin_value(order['cost'], quote_currency)})") - lines.append(f"*Average {wording} Price:* {cur_entry_average} " + lines.append(f"*Average {wording} Price:* {cur_entry_average:.8g} " f"({price_to_1st_entry:.2%} from 1st entry Rate)") lines.append(f"*Order filled:* {order['order_filled_date']}") @@ -633,11 +633,11 @@ class Telegram(RPCHandler): ]) lines.extend([ - "*Open Rate:* `{open_rate:.8f}`", - "*Close Rate:* `{close_rate:.8f}`" if r['close_rate'] else "", + "*Open Rate:* `{open_rate:.8g}`", + "*Close Rate:* `{close_rate:.8g}`" if r['close_rate'] else "", "*Open Date:* `{open_date}`", "*Close Date:* `{close_date}`" if r['close_date'] else "", - " \n*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", + " \n*Current Rate:* `{current_rate:.8g}`" if r['is_open'] else "", ("*Unrealized Profit:* " if r['is_open'] else "*Close Profit: *") + "`{profit_ratio:.2%}` `({profit_abs_r})`", ]) @@ -658,9 +658,9 @@ class Telegram(RPCHandler): "`({initial_stop_loss_ratio:.2%})`") # Adding stoploss and stoploss percentage only if it is not None - lines.append("*Stoploss:* `{stop_loss_abs:.8f}` " + + lines.append("*Stoploss:* `{stop_loss_abs:.8g}` " + ("`({stop_loss_ratio:.2%})`" if r['stop_loss_ratio'] else "")) - lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` " + lines.append("*Stoploss distance:* `{stoploss_current_dist:.8g}` " "`({stoploss_current_dist_ratio:.2%})`") if r['open_order']: lines.append( @@ -1114,7 +1114,9 @@ class Telegram(RPCHandler): async def _force_exit_action(self, trade_id): if trade_id != 'cancel': try: - self._rpc._rpc_force_exit(trade_id) + loop = asyncio.get_running_loop() + # Workaround to avoid nested loops + await loop.run_in_executor(None, self._rpc._rpc_force_exit, trade_id) except RPCException as e: await self._send_msg(str(e)) @@ -1140,7 +1142,11 @@ class Telegram(RPCHandler): async def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection): if pair != 'cancel': try: - self._rpc._rpc_force_entry(pair, price, order_side=order_side) + def _force_enter(): + self._rpc._rpc_force_entry(pair, price, order_side=order_side) + loop = asyncio.get_running_loop() + # Workaround to avoid nested loops + await loop.run_in_executor(None, _force_enter) except RPCException as e: logger.exception("Forcebuy error!") await self._send_msg(str(e), ParseMode.HTML) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index dfdfd31d8..d0655b504 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -1300,7 +1300,7 @@ class IStrategy(ABC, HyperStrategyMixin): timedout = (order.status == 'open' and order.order_date_utc < timeout_threshold) if timedout: return True - time_method = (self.check_exit_timeout if order.side == trade.exit_side + time_method = (self.check_exit_timeout if order.ft_order_side == trade.exit_side else self.check_entry_timeout) return strategy_safe_wrapper(time_method, diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 347efdda0..084cf2e89 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -232,7 +232,7 @@ class FreqaiExampleStrategy(IStrategy): # All indicators must be populated by feature_engineering_*() functions - # the model will return all labels created by user in `feature_engineering_*` + # the model will return all labels created by user in `set_freqai_targets()` # (& appended targets), an indication of whether or not the prediction should be accepted, # the target mean/std values for each of the labels created by user in # `set_freqai_targets()` for each training period. diff --git a/requirements-dev.txt b/requirements-dev.txt index 5822dd595..5239cd993 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,10 +7,10 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.270 +ruff==0.0.272 mypy==1.3.0 pre-commit==3.3.2 -pytest==7.3.1 +pytest==7.3.2 pytest-asyncio==0.21.0 pytest-cov==4.1.0 pytest-mock==3.10.0 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index de48a1da4..2672f9c38 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.28.1 -stable_baselines3==2.0.0a10 +stable_baselines3==2.0.0a13 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.65.0 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 424c74a9e..b9a50d92f 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,8 +5,7 @@ # Required for freqai scikit-learn==1.1.3 joblib==1.2.0 -catboost==1.1.1; sys_platform == 'darwin' and python_version < '3.9' -catboost==1.2; 'arm' not in platform_machine and (sys_platform != 'darwin' or python_version >= '3.9') +catboost==1.2; 'arm' not in platform_machine lightgbm==3.3.5 xgboost==1.7.5 tensorboard==2.13.0 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 87b1fd3c8..20c60afe0 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,4 +5,4 @@ scipy==1.10.1 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.12.0 +filelock==3.12.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index 8b9ad5bc4..72303efcb 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,4 +1,4 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.14.1 +plotly==5.15.0 diff --git a/requirements.txt b/requirements.txt index 118ad95e2..6d58dd0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,18 +2,18 @@ numpy==1.24.3 pandas==2.0.2 pandas-ta==0.3.14b -ccxt==3.1.23 +ccxt==3.1.34 cryptography==41.0.1; platform_machine != 'armv7l' cryptography==40.0.1; platform_machine == 'armv7l' aiohttp==3.8.4 -SQLAlchemy==2.0.15 +SQLAlchemy==2.0.16 python-telegram-bot==20.3 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 arrow==1.2.3 cachetools==5.3.1 requests==2.31.0 -urllib3==2.0.2 +urllib3==2.0.3 jsonschema==4.17.3 TA-Lib==0.4.26 technical==1.4.0 @@ -32,14 +32,14 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.10 # Properly format api responses -orjson==3.9.0 +orjson==3.9.1 # Notify systemd sdnotify==0.3.2 # API Server -fastapi==0.96.0 -pydantic==1.10.8 +fastapi==0.97.0 +pydantic==1.10.9 uvicorn==0.22.0 pyjwt==2.7.0 aiofiles==23.1.0 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 0772af269..f9c9858ed 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -314,6 +314,13 @@ class FtRestClient(): """ return self._get(f"strategy/{strategy}") + def pairlists_available(self): + """Lists available pairlist providers + + :return: json object + """ + return self._get("pairlists/available") + def plot_config(self): """Return plot configuration if the strategy defines one. diff --git a/setup.py b/setup.py index 9a04e07d0..996a8b8f9 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup plot = ['plotly>=4.0'] hyperopt = [ 'scipy', - 'scikit-learn', + 'scikit-learn<=1.1.3', 'scikit-optimize>=0.7.0', 'filelock', ] diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f022a0905..5fa2755d2 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -3489,6 +3489,27 @@ def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange.stoploss_adjust(1, {}, side="sell") +@pytest.mark.parametrize('side,ratio,expected', [ + ('sell', 0.99, 99.0), # Default + ('sell', 0.999, 99.9), + ('sell', 1, 100), + ('sell', 1.1, InvalidOrderException), + ('buy', 0.99, 101.0), # Default + ('buy', 0.999, 100.1), + ('buy', 1, 100), + ('buy', 1.1, InvalidOrderException), + ]) +def test__get_stop_limit_rate(default_conf_usdt, mocker, side, ratio, expected): + exchange = get_patched_exchange(mocker, default_conf_usdt, id='binance') + + order_types = {'stoploss_on_exchange_limit_ratio': ratio} + if isinstance(expected, type) and issubclass(expected, Exception): + with pytest.raises(expected): + exchange._get_stop_limit_rate(100, order_types, side) + else: + assert exchange._get_stop_limit_rate(100, order_types, side) == expected + + def test_merge_ft_has_dict(default_conf, mocker): mocker.patch.multiple(EXMS, _init_ccxt=MagicMock(return_value=MagicMock()), diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 3824eddb7..e8f059118 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -499,7 +499,11 @@ def test__set_leverage_okx(mocker, default_conf): assert api_mock.set_leverage.call_args_list[0][1]['params'] == { 'mgnMode': 'isolated', 'posSide': 'net'} + api_mock.set_leverage = MagicMock(side_effect=ccxt.NetworkError()) + exchange._lev_prep('BTC/USDT:USDT', 3.2, 'buy') + api_mock.fetch_leverage.call_count == 1 + api_mock.fetch_leverage = MagicMock(side_effect=ccxt.NetworkError()) ccxt_exceptionhandlers( mocker, default_conf, @@ -592,3 +596,25 @@ def test_stoploss_adjust_okx(mocker, default_conf, sl1, sl2, sl3, side): } assert exchange.stoploss_adjust(sl1, order, side=side) assert not exchange.stoploss_adjust(sl2, order, side=side) + + +def test_stoploss_cancel_okx(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='okx') + + exchange.cancel_order = MagicMock() + + exchange.cancel_stoploss_order('1234', 'ETH/USDT') + assert exchange.cancel_order.call_count == 1 + assert exchange.cancel_order.call_args_list[0][1]['order_id'] == '1234' + assert exchange.cancel_order.call_args_list[0][1]['pair'] == 'ETH/USDT' + assert exchange.cancel_order.call_args_list[0][1]['params'] == {'stop': True} + + +def test__get_stop_params_okx(mocker, default_conf): + default_conf['trading_mode'] = 'futures' + default_conf['margin_mode'] = 'isolated' + exchange = get_patched_exchange(mocker, default_conf, id='okx') + params = exchange._get_stop_params('ETH/USDT:USDT', 1500, 'sell') + + assert params['tdMode'] == 'isolated' + assert params['posSide'] == 'net' diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ac7904515..f793b1f9c 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1657,6 +1657,122 @@ def test_api_freqaimodels(botclient, tmpdir, mocker): ]} +def test_api_pairlists_available(botclient, tmpdir): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists/available") + + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/available") + assert_response(rc) + response = rc.json() + assert isinstance(response['pairlists'], list) + assert len(response['pairlists']) > 0 + + assert len([r for r in response['pairlists'] if r['name'] == 'AgeFilter']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'VolumePairList']) == 1 + assert len([r for r in response['pairlists'] if r['name'] == 'StaticPairList']) == 1 + + volumepl = [r for r in response['pairlists'] if r['name'] == 'VolumePairList'][0] + assert volumepl['is_pairlist_generator'] is True + assert len(volumepl['params']) > 1 + age_pl = [r for r in response['pairlists'] if r['name'] == 'AgeFilter'][0] + assert age_pl['is_pairlist_generator'] is False + assert len(volumepl['params']) > 2 + + +def test_api_pairlists_evaluate(botclient, tmpdir, mocker): + ftbot, client = botclient + ftbot.config['user_data_dir'] = Path(tmpdir) + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") + + assert_response(rc, 503) + assert rc.json()['detail'] == 'Bot is not in the correct state.' + + ftbot.config['runmode'] = RunMode.WEBSERVER + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/randomJob") + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' + + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC" + } + # Fail, already running + ApiBG.pairlist_running = True + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc, 400) + assert rc.json()['detail'] == 'Pairlist evaluation is already running.' + + # should start the run + ApiBG.pairlist_running = False + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + job_id = rc.json()['job_id'] + + rc = client_get(client, f"{BASE_URI}/background/RandomJob") + assert_response(rc, 404) + assert rc.json()['detail'] == 'Job not found.' + + rc = client_get(client, f"{BASE_URI}/background/{job_id}") + assert_response(rc) + response = rc.json() + assert response['job_id'] == job_id + assert response['job_category'] == 'pairlist' + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") + assert_response(rc) + response = rc.json() + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC',] + assert response['result']['length'] == 4 + + # Restart with additional filter, reducing the list to 2 + body['pairlists'].append({"method": "OffsetFilter", "number_assets": 2}) + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert rc.json()['status'] == 'Pairlist evaluation started in background.' + job_id = rc.json()['job_id'] + + rc = client_get(client, f"{BASE_URI}/pairlists/evaluate/{job_id}") + assert_response(rc) + response = rc.json() + assert response['result']['whitelist'] == ['ETH/BTC', 'LTC/BTC', ] + assert response['result']['length'] == 2 + # Patch __run_pairlists + plm = mocker.patch('freqtrade.rpc.api_server.api_background_tasks.__run_pairlist', + return_value=None) + body = { + "pairlists": [ + {"method": "StaticPairList", }, + ], + "blacklist": [ + ], + "stake_currency": "BTC", + "exchange": "randomExchange", + "trading_mode": "futures", + "margin_mode": "isolated", + } + rc = client_post(client, f"{BASE_URI}/pairlists/evaluate", body) + assert_response(rc) + assert plm.call_count == 1 + call_config = plm.call_args_list[0][0][1] + assert call_config['exchange']['name'] == 'randomExchange' + assert call_config['trading_mode'] == 'futures' + assert call_config['margin_mode'] == 'isolated' + + def test_list_available_pairs(botclient): ftbot, client = botclient diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 5b09abbd3..7808fb5c8 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,7 +1,5 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json -import logging -import sys import warnings from copy import deepcopy from pathlib import Path @@ -23,8 +21,6 @@ from freqtrade.configuration.load_config import (load_config_file, load_file, lo from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, _set_loggers, - setup_logging, setup_logging_pre) from tests.conftest import (CURRENT_TEST_STRATEGY, log_has, log_has_re, patched_configuration_load_config_file) @@ -594,7 +590,7 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) # Prevent setting loggers - mocker.patch('freqtrade.loggers._set_loggers', MagicMock) + mocker.patch('freqtrade.loggers.set_loggers', MagicMock) arglist = ['trade', '-vvv'] args = Arguments(arglist).get_parsed_arg() @@ -605,127 +601,6 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: assert log_has('Verbosity set to 3', caplog) -def test_set_loggers() -> None: - # Reset Logging to Debug, otherwise this fails randomly as it's set globally - logging.getLogger('requests').setLevel(logging.DEBUG) - logging.getLogger("urllib3").setLevel(logging.DEBUG) - logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) - logging.getLogger('telegram').setLevel(logging.DEBUG) - - previous_value1 = logging.getLogger('requests').level - previous_value2 = logging.getLogger('ccxt.base.exchange').level - previous_value3 = logging.getLogger('telegram').level - - _set_loggers() - - value1 = logging.getLogger('requests').level - assert previous_value1 is not value1 - assert value1 is logging.INFO - - value2 = logging.getLogger('ccxt.base.exchange').level - assert previous_value2 is not value2 - assert value2 is logging.INFO - - value3 = logging.getLogger('telegram').level - assert previous_value3 is not value3 - assert value3 is logging.INFO - - _set_loggers(verbosity=2) - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.INFO - assert logging.getLogger('telegram').level is logging.INFO - assert logging.getLogger('werkzeug').level is logging.INFO - - _set_loggers(verbosity=3, api_verbosity='error') - - assert logging.getLogger('requests').level is logging.DEBUG - assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG - assert logging.getLogger('telegram').level is logging.INFO - assert logging.getLogger('werkzeug').level is logging.ERROR - - -@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") -def test_set_loggers_syslog(): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'syslog:/dev/log', - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] - # setting up logging again should NOT cause the loggers to be added a second time. - setup_logging(config) - assert len(logger.handlers) == 3 - # reset handlers to not break pytest - logger.handlers = orig_handlers - - -@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") -def test_set_loggers_Filehandler(tmpdir): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - logfile = Path(tmpdir) / 'ft_logfile.log' - config = {'verbosity': 2, - 'logfile': str(logfile), - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - assert [x for x in logger.handlers if type(x) == FTBufferingHandler] - # setting up logging again should NOT cause the loggers to be added a second time. - setup_logging(config) - assert len(logger.handlers) == 3 - # reset handlers to not break pytest - if logfile.exists: - logfile.unlink() - logger.handlers = orig_handlers - - -@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") -def test_set_loggers_journald(mocker): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'journald', - } - - setup_logging_pre() - setup_logging(config) - assert len(logger.handlers) == 3 - assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] - assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] - # reset handlers to not break pytest - logger.handlers = orig_handlers - - -def test_set_loggers_journald_importerror(import_fails): - logger = logging.getLogger() - orig_handlers = logger.handlers - logger.handlers = [] - - config = {'verbosity': 2, - 'logfile': 'journald', - } - with pytest.raises(OperationalException, - match=r'You need the cysystemd python package.*'): - setup_logging(config) - logger.handlers = orig_handlers - - def test_set_logfile(default_conf, mocker, tmpdir): patched_configuration_load_config_file(mocker, default_conf) f = Path(tmpdir / "test_file.log") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 71f494372..24e726403 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1241,6 +1241,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_ 'type': 'stop_loss_limit', 'price': 3, 'average': 2, + 'filled': enter_order['amount'], + 'remaining': 0, 'amount': enter_order['amount'], }) mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hit) @@ -3009,8 +3011,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) freqtrade.strategy.check_exit_timeout = MagicMock(side_effect=KeyError) freqtrade.strategy.check_entry_timeout = MagicMock(side_effect=KeyError) @@ -3018,8 +3020,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) # Return True - sells! freqtrade.strategy.check_exit_timeout = MagicMock(return_value=True) @@ -3027,8 +3029,8 @@ def test_manage_open_orders_exit_usercustom( freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 - assert freqtrade.strategy.check_exit_timeout.call_count == 1 - assert freqtrade.strategy.check_entry_timeout.call_count == 0 + assert freqtrade.strategy.check_exit_timeout.call_count == (0 if is_short else 1) + assert freqtrade.strategy.check_entry_timeout.call_count == (1 if is_short else 0) trade = Trade.session.scalars(select(Trade)).first() # cancelling didn't succeed - order-id remains open. assert trade.open_order_id is not None diff --git a/tests/test_log_setup.py b/tests/test_log_setup.py new file mode 100644 index 000000000..a9be24723 --- /dev/null +++ b/tests/test_log_setup.py @@ -0,0 +1,130 @@ +import logging +import sys +from pathlib import Path + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.loggers import (FTBufferingHandler, FTStdErrStreamHandler, set_loggers, + setup_logging, setup_logging_pre) + + +def test_set_loggers() -> None: + # Reset Logging to Debug, otherwise this fails randomly as it's set globally + logging.getLogger('requests').setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.DEBUG) + logging.getLogger('ccxt.base.exchange').setLevel(logging.DEBUG) + logging.getLogger('telegram').setLevel(logging.DEBUG) + + previous_value1 = logging.getLogger('requests').level + previous_value2 = logging.getLogger('ccxt.base.exchange').level + previous_value3 = logging.getLogger('telegram').level + + set_loggers() + + value1 = logging.getLogger('requests').level + assert previous_value1 is not value1 + assert value1 is logging.INFO + + value2 = logging.getLogger('ccxt.base.exchange').level + assert previous_value2 is not value2 + assert value2 is logging.INFO + + value3 = logging.getLogger('telegram').level + assert previous_value3 is not value3 + assert value3 is logging.INFO + + set_loggers(verbosity=2) + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.INFO + assert logging.getLogger('telegram').level is logging.INFO + assert logging.getLogger('werkzeug').level is logging.INFO + + set_loggers(verbosity=3, api_verbosity='error') + + assert logging.getLogger('requests').level is logging.DEBUG + assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG + assert logging.getLogger('telegram').level is logging.INFO + assert logging.getLogger('werkzeug').level is logging.ERROR + + +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +def test_set_loggers_syslog(): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'syslog:/dev/log', + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + # setting up logging again should NOT cause the loggers to be added a second time. + setup_logging(config) + assert len(logger.handlers) == 3 + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +def test_set_loggers_Filehandler(tmpdir): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + logfile = Path(tmpdir) / 'ft_logfile.log' + config = {'verbosity': 2, + 'logfile': str(logfile), + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + assert [x for x in logger.handlers if type(x) == FTBufferingHandler] + # setting up logging again should NOT cause the loggers to be added a second time. + setup_logging(config) + assert len(logger.handlers) == 3 + # reset handlers to not break pytest + if logfile.exists: + logfile.unlink() + logger.handlers = orig_handlers + + +@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") +def test_set_loggers_journald(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + + setup_logging_pre() + setup_logging(config) + assert len(logger.handlers) == 3 + assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] + assert [x for x in logger.handlers if type(x) == FTStdErrStreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +def test_set_loggers_journald_importerror(import_fails): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + with pytest.raises(OperationalException, + match=r'You need the cysystemd python package.*'): + setup_logging(config) + logger.handlers = orig_handlers