CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-01-20 14:00:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9733 from Bloodhunter4rc/remotepairlist

Browse Source 
Remotepairlist: fix parsing json exception , add saving to a file of processed pairlist + docs
This commit is contained in:
Matthias
2024-01-27 12:09:37 +01:00
committed by GitHub
parent 523864601a 99b11c088b
commit e03e0e838b
3 changed files with 82 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
docs/includes/pairlists.md
View File

@@ -192,7 +192,8 @@ The RemotePairList is defined in the pairlists section of the configuration sett
"refresh_period": 1800,
"keep_pairlist_on_failure": true,
"read_timeout": 60,
"bearer_token": "my-bearer-token"
"bearer_token": "my-bearer-token",
"save_to_file": "user_data/filename.json"
}
]
```
@@ -207,6 +208,42 @@ In "append" mode, the retrieved pairlist is added to the original pairlist. All
The `pairlist_url` option specifies the URL of the remote server where the pairlist is located, or the path to a local file (if file:/// is prepended). This allows the user to use either a remote server or a local file as the source for the pairlist.
The `save_to_file` option, when provided with a valid filename, saves the processed pairlist to that file in JSON format. This option is optional, and by default, the pairlist is not saved to a file.
??? Example "Multi bot with shared pairlist example"
`save_to_file` can be used to save the pairlist to a file with Bot1:
```json
"pairlists": [
{
"method": "RemotePairList",
"mode": "whitelist",
"pairlist_url": "https://example.com/pairlist",
"number_assets": 10,
"refresh_period": 1800,
"keep_pairlist_on_failure": true,
"read_timeout": 60,
"save_to_file": "user_data/filename.json"
}
]
```
This saved pairlist file can be loaded by Bot2, or any additional bot with this configuration:
```json
"pairlists": [
{
"method": "RemotePairList",
"mode": "whitelist",
"pairlist_url": "file:///user_data/filename.json",
"number_assets": 10,
"refresh_period": 10,
"keep_pairlist_on_failure": true,
}
]
```
The user is responsible for providing a server or local file that returns a JSON object with the following structure:
```json

66
freqtrade/plugins/pairlist/RemotePairList.py
View File

@@ -52,6 +52,7 @@ class RemotePairList(IPairList):
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
self._init_done = False
self._save_to_file = self._pairlistconfig.get('save_to_file', None)
self._last_pairlist: List[Any] = list()
if self._mode not in ['whitelist', 'blacklist']:
@@ -136,6 +137,12 @@ class RemotePairList(IPairList):
"description": "Bearer token",
"help": "Bearer token - used for auth against the upstream service.",
},
"save_to_file": {
"type": "string",
"default": "",
"description": "Filename to save processed pairlist to.",
"help": "Specify a filename to save the processed pairlist in JSON format.",
},
}
def process_json(self, jsonparse) -> List[str]:
@@ -184,31 +191,26 @@ class RemotePairList(IPairList):
try:
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException(f'Error while processing JSON data: {type(e)}')
pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}')
else:
if self._init_done:
self.log_once(f'Error: RemotePairList is not of type JSON: '
f' {self._pairlist_url}', logger.info)
pairlist = self.return_last_pairlist()
else:
raise OperationalException('RemotePairList is not of type JSON, abort.')
pairlist = self._handle_error(f'RemotePairList is not of type JSON.'
f' {self._pairlist_url}')
except requests.exceptions.RequestException:
self.log_once(f'Was not able to fetch pairlist from:'
f' {self._pairlist_url}', logger.info)
pairlist = self.return_last_pairlist()
pairlist = self._handle_error(f'Was not able to fetch pairlist from:'
f' {self._pairlist_url}')
time_elapsed = 0
return pairlist, time_elapsed
def _handle_error(self, error: str) -> List[str]:
if self._init_done:
self.log_once("Error: " + error, logger.info)
return self.return_last_pairlist()
else:
raise OperationalException(error)
def gen_pairlist(self, tickers: Tickers) -> List[str]:
"""
Generate the pairlist
@@ -236,20 +238,15 @@ class RemotePairList(IPairList):
if file_path.exists():
with file_path.open() as json_file:
# Load the JSON data into a dictionary
jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)
try:
# Load the JSON data into a dictionary
jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException('Error while processing'
f'JSON data: {type(e)}')
pairlist = self._handle_error(f'processing JSON data: {type(e)}')
else:
raise ValueError(f"{self._pairlist_url} does not exist.")
pairlist = self._handle_error(f"{self._pairlist_url} does not exist.")
else:
# Fetch Pairlist from Remote URL
pairlist, time_elapsed = self.fetch_pairlist()
@@ -273,8 +270,23 @@ class RemotePairList(IPairList):
self._last_pairlist = list(pairlist)
if self._save_to_file:
self.save_pairlist(pairlist, self._save_to_file)
return pairlist
def save_pairlist(self, pairlist: List[str], filename: str) -> None:
pairlist_data = {
"pairs": pairlist
}
try:
file_path = Path(filename)
with file_path.open('w') as json_file:
rapidjson.dump(pairlist_data, json_file)
logger.info(f"Processed pairlist saved to {filename}")
except Exception as e:
logger.error(f"Error saving processed pairlist to {filename}: {e}")
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.

8
tests/plugins/test_remotepairlist.py
View File

@@ -82,7 +82,7 @@ def test_fetch_pairlist_mock_response_html(mocker, rpl_config):
remote_pairlist = RemotePairList(exchange, pairlistmanager, rpl_config,
rpl_config['pairlists'][0], 0)
with pytest.raises(OperationalException, match='RemotePairList is not of type JSON, abort.'):
with pytest.raises(OperationalException, match='RemotePairList is not of type JSON.'):
remote_pairlist.fetch_pairlist()
@@ -107,9 +107,11 @@ def test_fetch_pairlist_timeout_keep_last_pairlist(mocker, rpl_config, caplog):
rpl_config['pairlists'][0], 0)
remote_pairlist._last_pairlist = ["BTC/USDT", "ETH/USDT", "LTC/USDT"]
remote_pairlist._init_done = True
pairlist_url = rpl_config['pairlists'][0]['pairlist_url']
pairs, _time_elapsed = remote_pairlist.fetch_pairlist()
assert log_has(f"Was not able to fetch pairlist from: {remote_pairlist._pairlist_url}", caplog)
assert log_has(f'Error: Was not able to fetch pairlist from: ' f'{pairlist_url}', caplog)
assert log_has("Keeping last fetched pairlist", caplog)
assert pairs == ["BTC/USDT", "ETH/USDT", "LTC/USDT"]