From 13da8f936812bbfd255c4dfce4dd451dc9134c46 Mon Sep 17 00:00:00 2001 From: Daniel Goller Date: Mon, 9 Nov 2020 08:34:40 +0000 Subject: [PATCH 1/7] Added ConstPairList handler to skip validation of pairs if you want to backtest a pair that's not live any more, e.g. expiring contracts. --- freqtrade/constants.py | 2 +- freqtrade/pairlist/ConstPairList.py | 60 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 freqtrade/pairlist/ConstPairList.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index dc5384f6f..21308b2dc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -23,7 +23,7 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', +AVAILABLE_PAIRLISTS = ['ConstPairList', 'StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] diff --git a/freqtrade/pairlist/ConstPairList.py b/freqtrade/pairlist/ConstPairList.py new file mode 100644 index 000000000..e5b009c55 --- /dev/null +++ b/freqtrade/pairlist/ConstPairList.py @@ -0,0 +1,60 @@ +""" +Const Pair List provider + +Provides pair white list as it configured in config without checking for active markets +""" +import logging +from typing import Any, Dict, List + +from freqtrade.exceptions import OperationalException +from freqtrade.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class ConstPairList(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if self._pairlist_pos != 0: + raise OperationalException(f"{self.name} can only be used in the first position " + "in the list of Pairlist Handlers.") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + -> Please overwrite in subclasses + """ + return f"{self.name}" + + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + return self._config['exchange']['pair_whitelist'] + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + return pairlist From 916776bb53642b25c4be09bca82ef0c1467798d8 Mon Sep 17 00:00:00 2001 From: Daniel Goller Date: Mon, 9 Nov 2020 08:37:38 +0000 Subject: [PATCH 2/7] Option to skip exchange validation, required to backtest pairs that are not live on the exchange any more. --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 659ff59bc..baa379db1 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -115,7 +115,7 @@ class Exchange: logger.info('Using Exchange "%s"', self.name) - if validate: + if validate and not exchange_config.get('skip_validation'): # Check if timeframe is available self.validate_timeframes(config.get('timeframe')) From 2640dfee9387613480e66da00a4fac5b260c1e41 Mon Sep 17 00:00:00 2001 From: Daniel Goller Date: Thu, 12 Nov 2020 11:27:30 +0000 Subject: [PATCH 3/7] Revert "Added ConstPairList handler to skip validation of pairs if you want to backtest a pair that's not live any more, e.g. expiring contracts." This reverts commit 13da8f936812bbfd255c4dfce4dd451dc9134c46. --- freqtrade/constants.py | 2 +- freqtrade/pairlist/ConstPairList.py | 60 ----------------------------- 2 files changed, 1 insertion(+), 61 deletions(-) delete mode 100644 freqtrade/pairlist/ConstPairList.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 21308b2dc..dc5384f6f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -23,7 +23,7 @@ ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] -AVAILABLE_PAIRLISTS = ['ConstPairList', 'StaticPairList', 'VolumePairList', +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] diff --git a/freqtrade/pairlist/ConstPairList.py b/freqtrade/pairlist/ConstPairList.py deleted file mode 100644 index e5b009c55..000000000 --- a/freqtrade/pairlist/ConstPairList.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Const Pair List provider - -Provides pair white list as it configured in config without checking for active markets -""" -import logging -from typing import Any, Dict, List - -from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList - - -logger = logging.getLogger(__name__) - - -class ConstPairList(IPairList): - - def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], - pairlist_pos: int) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - - if self._pairlist_pos != 0: - raise OperationalException(f"{self.name} can only be used in the first position " - "in the list of Pairlist Handlers.") - - @property - def needstickers(self) -> bool: - """ - Boolean property defining if tickers are necessary. - If no Pairlist requires tickers, an empty List is passed - as tickers argument to filter_pairlist - """ - return False - - def short_desc(self) -> str: - """ - Short whitelist method description - used for startup-messages - -> Please overwrite in subclasses - """ - return f"{self.name}" - - def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: - """ - Generate the pairlist - :param cached_pairlist: Previously generated pairlist (cached) - :param tickers: Tickers (from exchange.get_tickers()). - :return: List of pairs - """ - return self._config['exchange']['pair_whitelist'] - - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ - Filters and sorts pairlist and returns the whitelist again. - Called on each bot iteration - please use internal caching if necessary - :param pairlist: pairlist to filter or sort - :param tickers: Tickers (from exchange.get_tickers()). May be cached. - :return: new whitelist - """ - return pairlist From 2424ac94c27c146a553aed362c04ac1850733381 Mon Sep 17 00:00:00 2001 From: Daniel Goller Date: Thu, 12 Nov 2020 11:29:46 +0000 Subject: [PATCH 4/7] skip the check for active markets with flag for existing StaticPairList --- freqtrade/pairlist/StaticPairList.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index aa6268ba3..3b6440763 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -24,6 +24,8 @@ class StaticPairList(IPairList): raise OperationalException(f"{self.name} can only be used in the first position " "in the list of Pairlist Handlers.") + self._allow_inactive = self._pairlistconfig.get('allow_inactive', False) + @property def needstickers(self) -> bool: """ @@ -47,7 +49,10 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + if self._allow_inactive: + return self._config['exchange']['pair_whitelist'] + else: + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ From 2d6bfe1592ab61258d523065cdf4b38cb3857521 Mon Sep 17 00:00:00 2001 From: Daniel Goller Date: Thu, 12 Nov 2020 11:32:45 +0000 Subject: [PATCH 5/7] only skip pair validation rather than all of it --- freqtrade/exchange/exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index baa379db1..0e982a06f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -115,7 +115,7 @@ class Exchange: logger.info('Using Exchange "%s"', self.name) - if validate and not exchange_config.get('skip_validation'): + if validate: # Check if timeframe is available self.validate_timeframes(config.get('timeframe')) @@ -124,7 +124,8 @@ class Exchange: # Check if all pairs are available self.validate_stakecurrency(config['stake_currency']) - self.validate_pairs(config['exchange']['pair_whitelist']) + if not exchange_config.get('skip_pair_validation'): + self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) self.validate_required_startup_candles(config.get('startup_candle_count', 0)) From f88fe5d950daf3ab68f9153648bf8d94f20593b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 19:14:43 +0100 Subject: [PATCH 6/7] Document new "allow_inactive" option --- docs/includes/pairlists.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index ae4ec818d..aebd084ab 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -35,6 +35,10 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis ], ``` +By default, only currently enabled pairs are allowed. +To skip pair validation against active markets, set `"allow_inactive": true` within the `StaticPairList` configuration. +This can be useful for backtesting expired pairs (like quarterly spot-markets). + #### Volume Pair List `VolumePairList` employs sorting/filtering of pairs by their trading volume. It selects `number_assets` top pairs with sorting based on the `sort_key` (which can only be `quoteVolume`). From 97e58a42f4caf1803af411937123461bda7ca244 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 19:17:31 +0100 Subject: [PATCH 7/7] Update documentation with new options --- docs/configuration.md | 1 + docs/includes/pairlists.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 47362e525..080ddd046 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -87,6 +87,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer +| `exchange.skip_pair_validation` | Skip pairlist validation on startup.
*Defaults to `false`
**Datatype:** Boolean | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information below](#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index aebd084ab..e6a9fc1a8 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -38,6 +38,7 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis By default, only currently enabled pairs are allowed. To skip pair validation against active markets, set `"allow_inactive": true` within the `StaticPairList` configuration. This can be useful for backtesting expired pairs (like quarterly spot-markets). +This option must be configured along with `exchange.skip_pair_validation` in the exchange configuration. #### Volume Pair List