From 83122fa58b8c6ebbe7bb68d19137077aabb95846 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Oct 2024 06:05:40 +0200 Subject: [PATCH] feat: add initial download-data api endpoints --- freqtrade/rpc/api_server/api_download_data.py | 99 +++++++++++++++++++ freqtrade/rpc/api_server/webserver.py | 6 ++ freqtrade/rpc/api_server/webserver_bgwork.py | 3 +- 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 freqtrade/rpc/api_server/api_download_data.py diff --git a/freqtrade/rpc/api_server/api_download_data.py b/freqtrade/rpc/api_server/api_download_data.py new file mode 100644 index 000000000..67d24e5e6 --- /dev/null +++ b/freqtrade/rpc/api_server/api_download_data.py @@ -0,0 +1,99 @@ +import logging +from copy import deepcopy + +from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi.exceptions import HTTPException + +from freqtrade.configuration.timerange import TimeRange +from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException +from freqtrade.persistence import FtNoDBContext +from freqtrade.rpc.api_server.api_pairlists import handleExchangePayload +from freqtrade.rpc.api_server.api_schemas import BgJobStarted, DownloadDataPayload +from freqtrade.rpc.api_server.deps import get_config, get_exchange +from freqtrade.rpc.api_server.webserver_bgwork import ApiBG + + +logger = logging.getLogger(__name__) + +# Private API, protected by authentication and webserver_mode dependency +router = APIRouter(tags=["download-data", "webserver"]) + + +def __run_download(job_id: str, config_loc: Config): + try: + ApiBG.jobs[job_id]["is_running"] = True + from freqtrade.data.history.history_utils import ( + refresh_backtest_ohlcv_data, + ) + from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist + + with FtNoDBContext(): + exchange = get_exchange(config_loc) + available_pairs = [ + p + for p in exchange.get_markets( + tradable_only=True, active_only=not config_loc.get("include_inactive") + ).keys() + ] + + expanded_pairs = dynamic_expand_pairlist(config_loc, available_pairs) + print(expanded_pairs) + timerange = TimeRange() + timerange = TimeRange.parse_timerange(config_loc.get("timerange")) + + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, + pairs=expanded_pairs, + timeframes=config_loc["timeframes"], + datadir=config_loc["datadir"], + timerange=timerange, + new_pairs_days=config_loc["new_pairs_days"], + erase=bool(config_loc.get("erase")), + data_format=config_loc["dataformat_ohlcv"], + trading_mode=config_loc.get("trading_mode", "spot"), + prepend=config_loc.get("prepend_data", False), + ) + ApiBG.jobs[job_id]["result"] = { + "pairs_not_available": pairs_not_available, + } + ApiBG.jobs[job_id]["status"] = "success" + except (OperationalException, Exception) as e: + logger.exception(e) + ApiBG.jobs[job_id]["error"] = str(e) + ApiBG.jobs[job_id]["status"] = "failed" + finally: + ApiBG.jobs[job_id]["is_running"] = False + ApiBG.download_data_running = False + + +@router.post("/download_data", response_model=BgJobStarted) +def pairlists_evaluate( + payload: DownloadDataPayload, background_tasks: BackgroundTasks, config=Depends(get_config) +): + if ApiBG.download_data_running: + raise HTTPException(status_code=400, detail="Pairlist evaluation is already running.") + config_loc = deepcopy(config) + config_loc["stake_currency"] = payload.stake_currency + config_loc["pairs"] = payload.pairs + config_loc["timeframes"] = payload.timeframes + handleExchangePayload(payload, config_loc) + print(payload) + + job_id = ApiBG.get_job_id() + + ApiBG.jobs[job_id] = { + "category": "download_data", + "status": "pending", + "progress": None, + "is_running": False, + "result": {}, + "error": None, + } + background_tasks.add_task(__run_download, job_id, config_loc) + ApiBG.download_data_running = True + + return { + "status": "Data Download started in background.", + "job_id": job_id, + } diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index aa3825df3..d69fca9f1 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -116,6 +116,7 @@ class ApiServer(RPCHandler): from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login from freqtrade.rpc.api_server.api_background_tasks import router as api_bg_tasks from freqtrade.rpc.api_server.api_backtest import router as api_backtest + from freqtrade.rpc.api_server.api_download_data import router as api_download_data from freqtrade.rpc.api_server.api_pairlists import router as api_pairlists from freqtrade.rpc.api_server.api_v1 import router as api_v1 from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public @@ -146,6 +147,11 @@ class ApiServer(RPCHandler): prefix="/api/v1", dependencies=[Depends(http_basic_or_jwt_token), Depends(is_webserver_mode)], ) + app.include_router( + api_download_data, + prefix="/api/v1", + dependencies=[Depends(http_basic_or_jwt_token), Depends(is_webserver_mode)], + ) app.include_router(ws_router, prefix="/api/v1") # UI Router MUST be last! app.include_router(router_ui, prefix="") diff --git a/freqtrade/rpc/api_server/webserver_bgwork.py b/freqtrade/rpc/api_server/webserver_bgwork.py index 642da632e..f57fc7eed 100644 --- a/freqtrade/rpc/api_server/webserver_bgwork.py +++ b/freqtrade/rpc/api_server/webserver_bgwork.py @@ -5,7 +5,7 @@ from freqtrade.exchange.exchange import Exchange class JobsContainer(TypedDict): - category: Literal["pairlist"] + category: Literal["pairlist", "download_data"] is_running: bool status: str progress: float | None @@ -32,6 +32,7 @@ class ApiBG: jobs: dict[str, JobsContainer] = {} # Pairlist evaluate things pairlist_running: bool = False + download_data_running: bool = False @staticmethod def get_job_id() -> str: