diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8e4b43178..ba9366493 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -192,7 +192,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett "refresh_period": 1800, "keep_pairlist_on_failure": true, "read_timeout": 60, - "bearer_token": "my-bearer-token" + "bearer_token": "my-bearer-token", + "save_to_file": "user_data/filename.json" } ] ``` @@ -207,6 +208,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. +The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file. + +??? Example "Multi bot with shared pairlist example" + + `save_to_file` can be used to save the pairlist to a file with Bot1: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "https://example.com/pairlist", + "number_assets": 10, + "refresh_period": 1800, + "keep_pairlist_on_failure": true, + "read_timeout": 60, + "save_to_file": "user_data/filename.json" + } + ] + ``` + + This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "file:///user_data/filename.json", + "number_assets": 10, + "refresh_period": 10, + "keep_pairlist_on_failure": true, + } + ] + ``` + The user is responsible for providing a server or local file that returns a JSON object with the following structure: ```json diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 2f03678e2..78e3c8351 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -52,6 +52,7 @@ class RemotePairList(IPairList): self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') self._init_done = False + self._save_to_file = self._pairlistconfig.get('save_to_file', None) self._last_pairlist: List[Any] = list() if self._mode not in ['whitelist', 'blacklist']: @@ -136,6 +137,12 @@ class RemotePairList(IPairList): "description": "Bearer token", "help": "Bearer token - used for auth against the upstream service.", }, + "save_to_file": { + "type": "string", + "default": "", + "description": "Filename to save processed pairlist to.", + "help": "Specify a filename to save the processed pairlist in JSON format.", + }, } def process_json(self, jsonparse) -> List[str]: @@ -184,31 +191,26 @@ class RemotePairList(IPairList): try: pairlist = self.process_json(jsonparse) except Exception as e: - - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - else: - raise OperationalException(f'Error while processing JSON data: {type(e)}') - + pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}') else: - if self._init_done: - self.log_once(f'Error: RemotePairList is not of type JSON: ' - f' {self._pairlist_url}', logger.info) - pairlist = self.return_last_pairlist() - else: - raise OperationalException('RemotePairList is not of type JSON, abort.') + pairlist = self._handle_error(f'RemotePairList is not of type JSON.' + f' {self._pairlist_url}') except requests.exceptions.RequestException: - self.log_once(f'Was not able to fetch pairlist from:' - f' {self._pairlist_url}', logger.info) - - pairlist = self.return_last_pairlist() + pairlist = self._handle_error(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}') time_elapsed = 0 return pairlist, time_elapsed + def _handle_error(self, error: str) -> List[str]: + if self._init_done: + self.log_once("Error: " + error, logger.info) + return self.return_last_pairlist() + else: + raise OperationalException(error) + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist @@ -236,20 +238,15 @@ class RemotePairList(IPairList): if file_path.exists(): with file_path.open() as json_file: - # Load the JSON data into a dictionary - jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) - try: + # Load the JSON data into a dictionary + jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) pairlist = self.process_json(jsonparse) except Exception as e: - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - else: - raise OperationalException('Error while processing' - f'JSON data: {type(e)}') + pairlist = self._handle_error(f'processing JSON data: {type(e)}') else: - raise ValueError(f"{self._pairlist_url} does not exist.") + pairlist = self._handle_error(f"{self._pairlist_url} does not exist.") + else: # Fetch Pairlist from Remote URL pairlist, time_elapsed = self.fetch_pairlist() @@ -273,8 +270,23 @@ class RemotePairList(IPairList): self._last_pairlist = list(pairlist) + if self._save_to_file: + self.save_pairlist(pairlist, self._save_to_file) + return pairlist + def save_pairlist(self, pairlist: List[str], filename: str) -> None: + pairlist_data = { + "pairs": pairlist + } + try: + file_path = Path(filename) + with file_path.open('w') as json_file: + rapidjson.dump(pairlist_data, json_file) + logger.info(f"Processed pairlist saved to {filename}") + except Exception as e: + logger.error(f"Error saving processed pairlist to {filename}: {e}") + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index b21763d47..9d407de9f 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -82,7 +82,7 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config): remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'): + with pytest.raises(OperationalException, match='RemotePairList is not of type JSON.'): remote_pairlist.fetch_pairlist() @@ -107,9 +107,11 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): rpl_config['pairlists'][0], 0) remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] - + remote_pairlist._init_done = True + pairlist_url = rpl_config['pairlists'][0]['pairlist_url'] pairs, _time_elapsed = remote_pairlist.fetch_pairlist() - assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) + + assert log_has(f'Error: Was not able to fetch pairlist from: ' f'{pairlist_url}', caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"]