Introduce background_job endpoints

freqtrade/rpc/api_server/api_schemas.py

@@ -27,6 +27,21 @@ class StatusMsg(BaseModel):
     status: str
 
 
+class BgJobStarted(StatusMsg):
+    job_id: str
+
+
+class BackgroundTaskStatus(BaseModel):
+    status: str
+    running: bool
+    progress: Optional[float]
+
+
+class BackgroundTaskResult(BaseModel):
+    error: Optional[str]
+    status: str
+
+
 class ResultMsg(BaseModel):
     result: str
 
@@ -376,10 +391,8 @@ class WhitelistResponse(BaseModel):
     method: List[str]
 
 
-class WhitelistEvaluateResponse(BaseModel):
+class WhitelistEvaluateResponse(BackgroundTaskResult):
     result: Optional[WhitelistResponse]
-    error: Optional[str]
-    status: str
 
 
 class DeleteTrade(BaseModel):

freqtrade/rpc/api_server/api_v1.py

@@ -11,16 +11,17 @@ from freqtrade.data.history import get_datahandler
 from freqtrade.enums import CandleType, TradingMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.rpc import RPC
-from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload,
+from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, BackgroundTaskStatus, Balances,
-                                                  BlacklistResponse, Count, Daily,
+                                                  BgJobStarted, BlacklistPayload, BlacklistResponse,
-                                                  DeleteLockRequest, DeleteTrade, ForceEnterPayload,
+                                                  Count, Daily, DeleteLockRequest, DeleteTrade,
-                                                  ForceEnterResponse, ForceExitPayload,
+                                                  ForceEnterPayload, ForceEnterResponse,
-                                                  FreqAIModelListResponse, Health, Locks, Logs,
+                                                  ForceExitPayload, FreqAIModelListResponse, Health,
-                                                  OpenTradeSchema, PairHistory, PairListsPayload,
+                                                  Locks, Logs, OpenTradeSchema, PairHistory,
-                                                  PairListsResponse, PerformanceEntry, Ping,
+                                                  PairListsPayload, PairListsResponse,
-                                                  PlotConfig, Profit, ResultMsg, ShowConfig, Stats,
+                                                  PerformanceEntry, Ping, PlotConfig, Profit,
-                                                  StatusMsg, StrategyListResponse, StrategyResponse,
+                                                  ResultMsg, ShowConfig, Stats, StatusMsg,
-                                                  SysInfo, Version, WhitelistEvaluateResponse,
+                                                  StrategyListResponse, StrategyResponse, SysInfo,
+                                                  Version, WhitelistEvaluateResponse,
                                                   WhitelistResponse)
 from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional
 from freqtrade.rpc.api_server.webserver_bgwork import ApiBG
@@ -333,26 +334,30 @@ def list_pairlists(config=Depends(get_config)):
     ]}
 
 
-def __run_pairlist(config_loc: Config):
+def __run_pairlist(job_id: str, config_loc: Config):
     try:
+
+        ApiBG.jobs[job_id]['is_running'] = True
         from freqtrade.plugins.pairlistmanager import PairListManager
 
         exchange = get_exchange(config_loc)
         pairlists = PairListManager(exchange, config_loc)
         pairlists.refresh_pairlist()
-        ApiBG.pairlist_result = {
+        ApiBG.jobs[job_id]['result'] = {
                 'method': pairlists.name_list,
                 'length': len(pairlists.whitelist),
                 'whitelist': pairlists.whitelist
             }
+        ApiBG.jobs[job_id]['status'] = 'success'
     except (OperationalException, Exception) as e:
         logger.exception(e)
-        ApiBG.pairlist_error = str(e)
+        ApiBG.jobs[job_id]['error'] = str(e)
     finally:
+        ApiBG.jobs[job_id]['is_running'] = False
         ApiBG.pairlist_running = False
 
 
-@router.post('/pairlists/evaluate', response_model=StatusMsg, tags=['pairlists'])
+@router.post('/pairlists/evaluate', response_model=BgJobStarted, tags=['pairlists'])
 def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTasks,
                        config=Depends(get_config)):
     if ApiBG.pairlist_running:
@@ -364,32 +369,60 @@ def pairlists_evaluate(payload: PairListsPayload, background_tasks: BackgroundTa
     # TODO: overwrite blacklist? make it optional and fall back to the one in config?
     # Outcome depends on the UI approach.
     config_loc['exchange']['pair_blacklist'] = payload.blacklist
-    ApiBG.pairlist_error = None
+    # Random job id
-    ApiBG.pairlist_result = {}
+    job_id = ApiBG.get_job_id()
-    background_tasks.add_task(__run_pairlist, config_loc)
+
+    ApiBG.jobs[job_id] = {
+        'category': 'pairlist',
+        'status': 'pending',
+        'progress': None,
+        'is_running': False,
+        'result': {},
+        'error': None,
+    }
+    ApiBG.running_jobs.append(job_id)
+    background_tasks.add_task(__run_pairlist, job_id, config_loc)
     ApiBG.pairlist_running = True
+
     return {
-        'status': 'Pairlist evaluation started in background.'
+        'status': 'Pairlist evaluation started in background.',
+        'job_id': job_id,
     }
 
 
-@router.get('/pairlists/evaluate', response_model=WhitelistEvaluateResponse, tags=['pairlists'])
+@router.get('/pairlists/evaluate/{jobid}', response_model=WhitelistEvaluateResponse,
-def pairlists_evaluate_get():
+            tags=['pairlists'])
+def pairlists_evaluate_get(jobid: str):
+    if not (job := ApiBG.jobs.get(jobid)):
+        raise HTTPException(status_code=404, detail='Job not found.')
 
-    if ApiBG.pairlist_running:
+    if job['is_running']:
-        return {'status': 'running'}
+        raise HTTPException(status_code=400, detail='Job not finished yet.')
-    if ApiBG.pairlist_error:
+
+    if error := job['error']:
         return {
             'status': 'failed',
-            'error': ApiBG.pairlist_error
+            'error': error,
         }
 
-    if not ApiBG.pairlist_result:
-        return {'status': 'pending'}
-
     return {
         'status': 'success',
-        'result': ApiBG.pairlist_result
+        'result': job['result'],
+    }
+
+
+@router.get('/background/{jobid}', response_model=BackgroundTaskStatus, tags=['webserver'])
+def background_job(jobid: str):
+    if not (job := ApiBG.jobs.get(jobid)):
+        raise HTTPException(status_code=404, detail='Job not found.')
+
+    return {
+        'job_id': jobid,
+        # 'type': job['job_type'],
+        'status': job['status'],
+        'running': job['is_running'],
+        'progress': job.get('progress'),
+        # 'job_error': job['error'],
     }
 
 

freqtrade/rpc/api_server/webserver_bgwork.py

@@ -1,5 +1,15 @@
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Literal, Optional, TypedDict
+from uuid import uuid4
+
+
+class JobsContainer(TypedDict):
+    category: Literal['pairlist']
+    is_running: bool
+    status: str
+    progress: Optional[float]
+    result: Any
+    error: Optional[str]
 
 
 class ApiBG():
@@ -14,7 +24,14 @@ class ApiBG():
     bgtask_running: bool = False
     # Exchange - only available in webserver mode.
     exchange = None
+
+    # Generic background jobs
+    running_jobs: List[str] = []
+    # TODO: Change this to TTLCache
+    jobs: Dict[str, JobsContainer] = {}
     # Pairlist evaluate things
-    pairlist_error: Optional[str] = None
     pairlist_running: bool = False
-    pairlist_result: Dict[str, Any] = {}
+
+    @staticmethod
+    def get_job_id() -> str:
+        return str(uuid4())