From fd216585236acab80e7f19fc84e9e403477df37d Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Fri, 26 Jan 2024 16:46:54 +0100 Subject: [PATCH 1/8] extend error except, add saving to a file of processed pairlist + docs --- docs/includes/pairlists.md | 5 ++++- freqtrade/plugins/pairlist/RemotePairList.py | 23 +++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8e4b43178..950f51f99 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -192,7 +192,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett "refresh_period": 1800, "keep_pairlist_on_failure": true, "read_timeout": 60, - "bearer_token": "my-bearer-token" + "bearer_token": "my-bearer-token", + "save_to_file": "user_data/filename.json" } ] ``` @@ -207,6 +208,8 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. +The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved. + The user is responsible for providing a server or local file that returns a JSON object with the following structure: ```json diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 2f03678e2..bf6bcd51b 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -52,6 +52,7 @@ class RemotePairList(IPairList): self._read_timeout = self._pairlistconfig.get('read_timeout', 60) self._bearer_token = self._pairlistconfig.get('bearer_token', '') self._init_done = False + self._save_to_file = self._pairlistconfig.get('save_to_file', None) self._last_pairlist: List[Any] = list() if self._mode not in ['whitelist', 'blacklist']: @@ -236,15 +237,15 @@ class RemotePairList(IPairList): if file_path.exists(): with file_path.open() as json_file: - # Load the JSON data into a dictionary - jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) - try: + # Load the JSON data into a dictionary + jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) pairlist = self.process_json(jsonparse) except Exception as e: if self._init_done: pairlist = self.return_last_pairlist() logger.warning(f'Error while processing JSON data: {type(e)}') + logger.debug(f'Error while processing JSON data: {e}') else: raise OperationalException('Error while processing' f'JSON data: {type(e)}') @@ -273,8 +274,24 @@ class RemotePairList(IPairList): self._last_pairlist = list(pairlist) + if self._save_to_file: + self.save_pairlist(pairlist, self._save_to_file) + return pairlist + def save_pairlist(self, pairlist: List[str], filename: str) -> None: + pairlist_data = { + "pairs": pairlist, + "refresh_period": self._refresh_period + } + try: + file_path = Path(filename) + with file_path.open('w') as json_file: + rapidjson.dump(pairlist_data, json_file) + logger.info(f"Processed pairlist saved to {filename}") + except Exception as e: + logger.error(f"Error saving processed pairlist to {filename}: {e}") + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. From 027ce4337d128af87fead556b5991322d7747031 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Fri, 26 Jan 2024 17:08:38 +0100 Subject: [PATCH 2/8] refresh_period not necessary for a local file --- freqtrade/plugins/pairlist/RemotePairList.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index bf6bcd51b..079ff49b6 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -281,8 +281,7 @@ class RemotePairList(IPairList): def save_pairlist(self, pairlist: List[str], filename: str) -> None: pairlist_data = { - "pairs": pairlist, - "refresh_period": self._refresh_period + "pairs": pairlist } try: file_path = Path(filename) From f0562c391ca8e37c951168b8655126d57538fe53 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Fri, 26 Jan 2024 18:32:46 +0100 Subject: [PATCH 3/8] remove debug, reduce duplicate code -> init_check, add docs example for save_to_file --- docs/includes/pairlists.md | 37 +++++++++++++++++- freqtrade/plugins/pairlist/RemotePairList.py | 41 ++++++++------------ 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 950f51f99..a07fe5d29 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -208,7 +208,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist. -The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved. +The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file. + +??? Note + Example: + + `save_to_file` can be used to save the pairlist to a file with Bot1: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "https://example.com/pairlist", + "number_assets": 10, + "refresh_period": 1800, + "keep_pairlist_on_failure": true, + "read_timeout": 60, + "save_to_file": "user_data/filename.json" + } + ] + ``` + + This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration: + + ```json + "pairlists": [ + { + "method": "RemotePairList", + "mode": "whitelist", + "pairlist_url": "file:///user_data/filename.json", + "number_assets": 10, + "refresh_period": 10, + "keep_pairlist_on_failure": true, + } + ] + ``` The user is responsible for providing a server or local file that returns a JSON object with the following structure: diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 079ff49b6..ea0652575 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -185,31 +185,27 @@ class RemotePairList(IPairList): try: pairlist = self.process_json(jsonparse) except Exception as e: - - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - else: - raise OperationalException(f'Error while processing JSON data: {type(e)}') - + pairlist = self.init_check(f'Failed processing JSON data: {type(e)}') else: - if self._init_done: - self.log_once(f'Error: RemotePairList is not of type JSON: ' - f' {self._pairlist_url}', logger.info) - pairlist = self.return_last_pairlist() - else: - raise OperationalException('RemotePairList is not of type JSON, abort.') + pairlist = self.init_check(f'RemotePairList is not of type JSON: ' + f' {self._pairlist_url}') except requests.exceptions.RequestException: - self.log_once(f'Was not able to fetch pairlist from:' - f' {self._pairlist_url}', logger.info) - - pairlist = self.return_last_pairlist() + pairlist = self.init_check(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}') time_elapsed = 0 return pairlist, time_elapsed + def init_check(self, error: str): + if self._init_done: + self.log_once("Error: " + error, logger.info) + pairlist = self.return_last_pairlist() + else: + raise OperationalException(error) + return pairlist + def gen_pairlist(self, tickers: Tickers) -> List[str]: """ Generate the pairlist @@ -242,15 +238,10 @@ class RemotePairList(IPairList): jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) pairlist = self.process_json(jsonparse) except Exception as e: - if self._init_done: - pairlist = self.return_last_pairlist() - logger.warning(f'Error while processing JSON data: {type(e)}') - logger.debug(f'Error while processing JSON data: {e}') - else: - raise OperationalException('Error while processing' - f'JSON data: {type(e)}') + pairlist = self.init_check(f'processing JSON data: {type(e)}') else: - raise ValueError(f"{self._pairlist_url} does not exist.") + pairlist = self.init_check(f"{self._pairlist_url} does not exist.") + else: # Fetch Pairlist from Remote URL pairlist, time_elapsed = self.fetch_pairlist() From dd3fbfcfdaffe89312d99c419cd991e70f34eb16 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Fri, 26 Jan 2024 18:56:47 +0100 Subject: [PATCH 4/8] + return type --- freqtrade/plugins/pairlist/RemotePairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index ea0652575..0b16b7872 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -198,7 +198,7 @@ class RemotePairList(IPairList): return pairlist, time_elapsed - def init_check(self, error: str): + def init_check(self, error: str) -> List[str]: if self._init_done: self.log_once("Error: " + error, logger.info) pairlist = self.return_last_pairlist() From c398504f23f270919854f96c39fe4060979eb9ae Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Fri, 26 Jan 2024 20:55:24 +0100 Subject: [PATCH 5/8] fix tests --- freqtrade/plugins/pairlist/RemotePairList.py | 2 +- tests/plugins/test_remotepairlist.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 0b16b7872..65dc626ab 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -187,7 +187,7 @@ class RemotePairList(IPairList): except Exception as e: pairlist = self.init_check(f'Failed processing JSON data: {type(e)}') else: - pairlist = self.init_check(f'RemotePairList is not of type JSON: ' + pairlist = self.init_check(f'RemotePairList is not of type JSON.' f' {self._pairlist_url}') except requests.exceptions.RequestException: diff --git a/tests/plugins/test_remotepairlist.py b/tests/plugins/test_remotepairlist.py index b21763d47..9d407de9f 100644 --- a/tests/plugins/test_remotepairlist.py +++ b/tests/plugins/test_remotepairlist.py @@ -82,7 +82,7 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config): remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config, rpl_config['pairlists'][0], 0) - with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'): + with pytest.raises(OperationalException, match='RemotePairList is not of type JSON.'): remote_pairlist.fetch_pairlist() @@ -107,9 +107,11 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog): rpl_config['pairlists'][0], 0) remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"] - + remote_pairlist._init_done = True + pairlist_url = rpl_config['pairlists'][0]['pairlist_url'] pairs, _time_elapsed = remote_pairlist.fetch_pairlist() - assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog) + + assert log_has(f'Error: Was not able to fetch pairlist from: ' f'{pairlist_url}', caplog) assert log_has("Keeping last fetched pairlist", caplog) assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"] From a20fe8cd094134b3aacbcb41013a75eaca7485c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 08:14:48 +0100 Subject: [PATCH 6/8] Update docs example box --- docs/includes/pairlists.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index a07fe5d29..ba9366493 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -210,8 +210,7 @@ The `pairlist_url` option specifies the URL of the remote server where the pairl The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file. -??? Note - Example: +??? Example "Multi bot with shared pairlist example" `save_to_file` can be used to save the pairlist to a file with Bot1: From f42fd2580041869abd531964bce84f42be1972a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 08:15:05 +0100 Subject: [PATCH 7/8] Improve function naming better reflecting what it aims to do --- freqtrade/plugins/pairlist/RemotePairList.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index 65dc626ab..cc573e701 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -185,26 +185,25 @@ class RemotePairList(IPairList): try: pairlist = self.process_json(jsonparse) except Exception as e: - pairlist = self.init_check(f'Failed processing JSON data: {type(e)}') + pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}') else: - pairlist = self.init_check(f'RemotePairList is not of type JSON.' - f' {self._pairlist_url}') + pairlist = self._handle_error(f'RemotePairList is not of type JSON.' + f' {self._pairlist_url}') except requests.exceptions.RequestException: - pairlist = self.init_check(f'Was not able to fetch pairlist from:' - f' {self._pairlist_url}') + pairlist = self._handle_error(f'Was not able to fetch pairlist from:' + f' {self._pairlist_url}') time_elapsed = 0 return pairlist, time_elapsed - def init_check(self, error: str) -> List[str]: + def _handle_error(self, error: str) -> List[str]: if self._init_done: self.log_once("Error: " + error, logger.info) - pairlist = self.return_last_pairlist() + return self.return_last_pairlist() else: raise OperationalException(error) - return pairlist def gen_pairlist(self, tickers: Tickers) -> List[str]: """ @@ -238,9 +237,9 @@ class RemotePairList(IPairList): jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE) pairlist = self.process_json(jsonparse) except Exception as e: - pairlist = self.init_check(f'processing JSON data: {type(e)}') + pairlist = self._handle_error(f'processing JSON data: {type(e)}') else: - pairlist = self.init_check(f"{self._pairlist_url} does not exist.") + pairlist = self._handle_error(f"{self._pairlist_url} does not exist.") else: # Fetch Pairlist from Remote URL From 99b11c088b32ff17b0a3ebef6789066577037f49 Mon Sep 17 00:00:00 2001 From: Bloodhunter4rc Date: Sat, 27 Jan 2024 08:18:18 +0100 Subject: [PATCH 8/8] add available_parameters --- freqtrade/plugins/pairlist/RemotePairList.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/plugins/pairlist/RemotePairList.py b/freqtrade/plugins/pairlist/RemotePairList.py index cc573e701..78e3c8351 100644 --- a/freqtrade/plugins/pairlist/RemotePairList.py +++ b/freqtrade/plugins/pairlist/RemotePairList.py @@ -137,6 +137,12 @@ class RemotePairList(IPairList): "description": "Bearer token", "help": "Bearer token - used for auth against the upstream service.", }, + "save_to_file": { + "type": "string", + "default": "", + "description": "Filename to save processed pairlist to.", + "help": "Specify a filename to save the processed pairlist in JSON format.", + }, } def process_json(self, jsonparse) -> List[str]: