mirror of
https://github.com/docling-project/docling-serve.git
synced 2025-11-29 08:33:50 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce15e0302b | ||
|
|
ecb1874a50 | ||
|
|
1333f71c9c | ||
|
|
ec594d84fe | ||
|
|
3771c1b554 |
12
CHANGELOG.md
12
CHANGELOG.md
@@ -1,3 +1,15 @@
|
||||
## [v1.1.0](https://github.com/docling-project/docling-serve/releases/tag/v1.1.0) - 2025-07-30
|
||||
|
||||
### Feature
|
||||
|
||||
* Add docling-mcp in the distribution ([#290](https://github.com/docling-project/docling-serve/issues/290)) ([`ecb1874`](https://github.com/docling-project/docling-serve/commit/ecb1874a507bef83d102e0e031e49fed34298637))
|
||||
* Add 3.0 openapi endpoint ([#287](https://github.com/docling-project/docling-serve/issues/287)) ([`ec594d8`](https://github.com/docling-project/docling-serve/commit/ec594d84fe36df23e7d010a2fcf769856c43600b))
|
||||
* Add new source and target ([#270](https://github.com/docling-project/docling-serve/issues/270)) ([`3771c1b`](https://github.com/docling-project/docling-serve/commit/3771c1b55403bd51966d07d8f760d5c4fbcc1760))
|
||||
|
||||
### Fix
|
||||
|
||||
* Referenced paths relative to zip root ([#289](https://github.com/docling-project/docling-serve/issues/289)) ([`1333f71`](https://github.com/docling-project/docling-serve/commit/1333f71c9c6495342b2169d574e921f828446f15))
|
||||
|
||||
## [v1.0.1](https://github.com/docling-project/docling-serve/releases/tag/v1.0.1) - 2025-07-21
|
||||
|
||||
### Fix
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import importlib.metadata
|
||||
import logging
|
||||
import shutil
|
||||
@@ -24,7 +25,7 @@ from fastapi.openapi.docs import (
|
||||
get_swagger_ui_html,
|
||||
get_swagger_ui_oauth2_redirect_html,
|
||||
)
|
||||
from fastapi.responses import RedirectResponse
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from scalar_fastapi import get_scalar_api_reference
|
||||
|
||||
@@ -34,8 +35,13 @@ from docling_jobkit.datamodel.callback import (
|
||||
ProgressCallbackResponse,
|
||||
)
|
||||
from docling_jobkit.datamodel.http_inputs import FileSource, HttpSource
|
||||
from docling_jobkit.datamodel.s3_coords import S3Coordinates
|
||||
from docling_jobkit.datamodel.task import Task, TaskSource
|
||||
from docling_jobkit.datamodel.task_targets import InBodyTarget, TaskTarget, ZipTarget
|
||||
from docling_jobkit.datamodel.task_targets import (
|
||||
InBodyTarget,
|
||||
TaskTarget,
|
||||
ZipTarget,
|
||||
)
|
||||
from docling_jobkit.orchestrators.base_orchestrator import (
|
||||
BaseOrchestrator,
|
||||
ProgressInvalid,
|
||||
@@ -47,6 +53,7 @@ from docling_serve.datamodel.requests import (
|
||||
ConvertDocumentsRequest,
|
||||
FileSourceRequest,
|
||||
HttpSourceRequest,
|
||||
S3SourceRequest,
|
||||
TargetName,
|
||||
)
|
||||
from docling_serve.datamodel.responses import (
|
||||
@@ -54,6 +61,7 @@ from docling_serve.datamodel.responses import (
|
||||
ConvertDocumentResponse,
|
||||
HealthCheckResponse,
|
||||
MessageKind,
|
||||
PresignedUrlConvertDocumentResponse,
|
||||
TaskStatusResponse,
|
||||
WebsocketMessage,
|
||||
)
|
||||
@@ -246,6 +254,8 @@ def create_app(): # noqa: C901
|
||||
sources.append(FileSource.model_validate(s))
|
||||
elif isinstance(s, HttpSourceRequest):
|
||||
sources.append(HttpSource.model_validate(s))
|
||||
elif isinstance(s, S3SourceRequest):
|
||||
sources.append(S3Coordinates.model_validate(s))
|
||||
|
||||
task = await orchestrator.enqueue(
|
||||
sources=sources,
|
||||
@@ -286,10 +296,79 @@ def create_app(): # noqa: C901
|
||||
if elapsed_time > docling_serve_settings.max_sync_wait:
|
||||
return False
|
||||
|
||||
##########################################
|
||||
# Downgrade openapi 3.1 to 3.0.x helpers #
|
||||
##########################################
|
||||
|
||||
def ensure_array_items(schema):
|
||||
"""Ensure that array items are defined."""
|
||||
if "type" in schema and schema["type"] == "array":
|
||||
if "items" not in schema or schema["items"] is None:
|
||||
schema["items"] = {"type": "string"}
|
||||
elif isinstance(schema["items"], dict):
|
||||
if "type" not in schema["items"]:
|
||||
schema["items"]["type"] = "string"
|
||||
|
||||
def handle_discriminators(schema):
|
||||
"""Ensure that discriminator properties are included in required."""
|
||||
if "discriminator" in schema and "propertyName" in schema["discriminator"]:
|
||||
prop = schema["discriminator"]["propertyName"]
|
||||
if "properties" in schema and prop in schema["properties"]:
|
||||
if "required" not in schema:
|
||||
schema["required"] = []
|
||||
if prop not in schema["required"]:
|
||||
schema["required"].append(prop)
|
||||
|
||||
def handle_properties(schema):
|
||||
"""Ensure that property 'kind' is included in required."""
|
||||
if "properties" in schema and "kind" in schema["properties"]:
|
||||
if "required" not in schema:
|
||||
schema["required"] = []
|
||||
if "kind" not in schema["required"]:
|
||||
schema["required"].append("kind")
|
||||
|
||||
# Downgrade openapi 3.1 to 3.0.x
|
||||
def downgrade_openapi31_to_30(spec):
|
||||
def strip_unsupported(obj):
|
||||
if isinstance(obj, dict):
|
||||
obj = {
|
||||
k: strip_unsupported(v)
|
||||
for k, v in obj.items()
|
||||
if k not in ("const", "examples", "prefixItems")
|
||||
}
|
||||
|
||||
handle_discriminators(obj)
|
||||
ensure_array_items(obj)
|
||||
|
||||
# Check for oneOf and anyOf to handle nested schemas
|
||||
for key in ["oneOf", "anyOf"]:
|
||||
if key in obj:
|
||||
for sub in obj[key]:
|
||||
handle_discriminators(sub)
|
||||
ensure_array_items(sub)
|
||||
|
||||
return obj
|
||||
elif isinstance(obj, list):
|
||||
return [strip_unsupported(i) for i in obj]
|
||||
return obj
|
||||
|
||||
if "components" in spec and "schemas" in spec["components"]:
|
||||
for schema_name, schema in spec["components"]["schemas"].items():
|
||||
handle_properties(schema)
|
||||
|
||||
return strip_unsupported(copy.deepcopy(spec))
|
||||
|
||||
#############################
|
||||
# API Endpoints definitions #
|
||||
#############################
|
||||
|
||||
@app.get("/openapi-3.0.json")
|
||||
def openapi_30():
|
||||
spec = app.openapi()
|
||||
downgraded = downgrade_openapi31_to_30(spec)
|
||||
downgraded["openapi"] = "3.0.3"
|
||||
return JSONResponse(downgraded)
|
||||
|
||||
# Favicon
|
||||
@app.get("/favicon.ico", include_in_schema=False)
|
||||
async def favicon():
|
||||
@@ -443,7 +522,8 @@ def create_app(): # noqa: C901
|
||||
orchestrator: Annotated[BaseOrchestrator, Depends(get_async_orchestrator)],
|
||||
task_id: str,
|
||||
wait: Annotated[
|
||||
float, Query(help="Number of seconds to wait for a completed status.")
|
||||
float,
|
||||
Query(description="Number of seconds to wait for a completed status."),
|
||||
] = 0.0,
|
||||
):
|
||||
try:
|
||||
@@ -525,7 +605,7 @@ def create_app(): # noqa: C901
|
||||
# Task result
|
||||
@app.get(
|
||||
"/v1/result/{task_id}",
|
||||
response_model=ConvertDocumentResponse,
|
||||
response_model=ConvertDocumentResponse | PresignedUrlConvertDocumentResponse,
|
||||
responses={
|
||||
200: {
|
||||
"content": {"application/zip": {}},
|
||||
|
||||
@@ -1,12 +1,21 @@
|
||||
import enum
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
from pydantic_core import PydanticCustomError
|
||||
from typing_extensions import Self
|
||||
|
||||
from docling_jobkit.datamodel.http_inputs import FileSource, HttpSource
|
||||
from docling_jobkit.datamodel.task_targets import InBodyTarget, TaskTarget, ZipTarget
|
||||
from docling_jobkit.datamodel.s3_coords import S3Coordinates
|
||||
from docling_jobkit.datamodel.task_targets import (
|
||||
InBodyTarget,
|
||||
S3Target,
|
||||
TaskTarget,
|
||||
ZipTarget,
|
||||
)
|
||||
|
||||
from docling_serve.datamodel.convert import ConvertDocumentsRequestOptions
|
||||
from docling_serve.settings import AsyncEngine, docling_serve_settings
|
||||
|
||||
## Sources
|
||||
|
||||
@@ -19,6 +28,10 @@ class HttpSourceRequest(HttpSource):
|
||||
kind: Literal["http"] = "http"
|
||||
|
||||
|
||||
class S3SourceRequest(S3Coordinates):
|
||||
kind: Literal["s3"] = "s3"
|
||||
|
||||
|
||||
## Multipart targets
|
||||
class TargetName(str, enum.Enum):
|
||||
INBODY = InBodyTarget().kind
|
||||
@@ -27,7 +40,7 @@ class TargetName(str, enum.Enum):
|
||||
|
||||
## Aliases
|
||||
SourceRequestItem = Annotated[
|
||||
FileSourceRequest | HttpSourceRequest, Field(discriminator="kind")
|
||||
FileSourceRequest | HttpSourceRequest | S3SourceRequest, Field(discriminator="kind")
|
||||
]
|
||||
|
||||
|
||||
@@ -36,3 +49,24 @@ class ConvertDocumentsRequest(BaseModel):
|
||||
options: ConvertDocumentsRequestOptions = ConvertDocumentsRequestOptions()
|
||||
sources: list[SourceRequestItem]
|
||||
target: TaskTarget = InBodyTarget()
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_s3_source_and_target(self) -> Self:
|
||||
for source in self.sources:
|
||||
if isinstance(source, S3SourceRequest):
|
||||
if docling_serve_settings.eng_kind != AsyncEngine.KFP:
|
||||
raise PydanticCustomError(
|
||||
"error source", 'source kind "s3" requires engine kind "KFP"'
|
||||
)
|
||||
if self.target.kind != "s3":
|
||||
raise PydanticCustomError(
|
||||
"error source", 'source kind "s3" requires target kind "s3"'
|
||||
)
|
||||
if isinstance(self.target, S3Target):
|
||||
for source in self.sources:
|
||||
if isinstance(source, S3SourceRequest):
|
||||
return self
|
||||
raise PydanticCustomError(
|
||||
"error target", 'target kind "s3" requires source kind "s3"'
|
||||
)
|
||||
return self
|
||||
|
||||
@@ -35,6 +35,11 @@ class ConvertDocumentResponse(BaseModel):
|
||||
timings: dict[str, ProfilingItem] = {}
|
||||
|
||||
|
||||
class PresignedUrlConvertDocumentResponse(BaseModel):
|
||||
status: ConversionStatus
|
||||
processing_time: float
|
||||
|
||||
|
||||
class ConvertDocumentErrorResponse(BaseModel):
|
||||
status: ConversionStatus
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
@@ -15,12 +16,16 @@ from docling.datamodel.document import ConversionResult, ConversionStatus
|
||||
from docling_core.types.doc import ImageRefMode
|
||||
from docling_jobkit.datamodel.convert import ConvertDocumentsOptions
|
||||
from docling_jobkit.datamodel.task import Task
|
||||
from docling_jobkit.datamodel.task_targets import InBodyTarget, TaskTarget
|
||||
from docling_jobkit.datamodel.task_targets import InBodyTarget, PutTarget, TaskTarget
|
||||
from docling_jobkit.orchestrators.base_orchestrator import (
|
||||
BaseOrchestrator,
|
||||
)
|
||||
|
||||
from docling_serve.datamodel.responses import ConvertDocumentResponse, DocumentResponse
|
||||
from docling_serve.datamodel.responses import (
|
||||
ConvertDocumentResponse,
|
||||
DocumentResponse,
|
||||
PresignedUrlConvertDocumentResponse,
|
||||
)
|
||||
from docling_serve.settings import docling_serve_settings
|
||||
from docling_serve.storage import get_scratch
|
||||
|
||||
@@ -79,11 +84,17 @@ def _export_documents_as_files(
|
||||
export_doctags: bool,
|
||||
image_export_mode: ImageRefMode,
|
||||
md_page_break_placeholder: str,
|
||||
):
|
||||
) -> ConversionStatus:
|
||||
success_count = 0
|
||||
failure_count = 0
|
||||
|
||||
# Default failure in case results is empty
|
||||
conv_result = ConversionStatus.FAILURE
|
||||
|
||||
artifacts_dir = Path("artifacts/") # will be relative to the fname
|
||||
|
||||
for conv_res in conv_results:
|
||||
conv_result = conv_res.status
|
||||
if conv_res.status == ConversionStatus.SUCCESS:
|
||||
success_count += 1
|
||||
doc_filename = conv_res.input.file.stem
|
||||
@@ -93,7 +104,9 @@ def _export_documents_as_files(
|
||||
fname = output_dir / f"{doc_filename}.json"
|
||||
_log.info(f"writing JSON output to {fname}")
|
||||
conv_res.document.save_as_json(
|
||||
filename=fname, image_mode=image_export_mode
|
||||
filename=fname,
|
||||
image_mode=image_export_mode,
|
||||
artifacts_dir=artifacts_dir,
|
||||
)
|
||||
|
||||
# Export HTML format:
|
||||
@@ -101,7 +114,9 @@ def _export_documents_as_files(
|
||||
fname = output_dir / f"{doc_filename}.html"
|
||||
_log.info(f"writing HTML output to {fname}")
|
||||
conv_res.document.save_as_html(
|
||||
filename=fname, image_mode=image_export_mode
|
||||
filename=fname,
|
||||
image_mode=image_export_mode,
|
||||
artifacts_dir=artifacts_dir,
|
||||
)
|
||||
|
||||
# Export Text format:
|
||||
@@ -120,6 +135,7 @@ def _export_documents_as_files(
|
||||
_log.info(f"writing Markdown output to {fname}")
|
||||
conv_res.document.save_as_markdown(
|
||||
filename=fname,
|
||||
artifacts_dir=artifacts_dir,
|
||||
image_mode=image_export_mode,
|
||||
page_break_placeholder=md_page_break_placeholder or None,
|
||||
)
|
||||
@@ -138,6 +154,7 @@ def _export_documents_as_files(
|
||||
f"Processed {success_count + failure_count} docs, "
|
||||
f"of which {failure_count} failed"
|
||||
)
|
||||
return conv_result
|
||||
|
||||
|
||||
def process_results(
|
||||
@@ -145,7 +162,7 @@ def process_results(
|
||||
target: TaskTarget,
|
||||
conv_results: Iterable[ConversionResult],
|
||||
work_dir: Path,
|
||||
) -> Union[ConvertDocumentResponse, FileResponse]:
|
||||
) -> Union[ConvertDocumentResponse, FileResponse, PresignedUrlConvertDocumentResponse]:
|
||||
# Let's start by processing the documents
|
||||
try:
|
||||
start_time = time.monotonic()
|
||||
@@ -169,7 +186,9 @@ def process_results(
|
||||
)
|
||||
|
||||
# We have some results, let's prepare the response
|
||||
response: Union[FileResponse, ConvertDocumentResponse]
|
||||
response: Union[
|
||||
FileResponse, ConvertDocumentResponse, PresignedUrlConvertDocumentResponse
|
||||
]
|
||||
|
||||
# Booleans to know what to export
|
||||
export_json = OutputFormat.JSON in conversion_options.to_formats
|
||||
@@ -209,7 +228,7 @@ def process_results(
|
||||
os.getpid()
|
||||
|
||||
# Export the documents
|
||||
_export_documents_as_files(
|
||||
conv_result = _export_documents_as_files(
|
||||
conv_results=conv_results,
|
||||
output_dir=output_dir,
|
||||
export_json=export_json,
|
||||
@@ -236,9 +255,24 @@ def process_results(
|
||||
# Output directory
|
||||
# background_tasks.add_task(shutil.rmtree, work_dir, ignore_errors=True)
|
||||
|
||||
response = FileResponse(
|
||||
file_path, filename=file_path.name, media_type="application/zip"
|
||||
)
|
||||
if isinstance(target, PutTarget):
|
||||
try:
|
||||
with open(file_path, "rb") as file_data:
|
||||
r = httpx.put(str(target.url), files={"file": file_data})
|
||||
r.raise_for_status()
|
||||
response = PresignedUrlConvertDocumentResponse(
|
||||
status=conv_result,
|
||||
processing_time=processing_time,
|
||||
)
|
||||
except Exception as exc:
|
||||
_log.error("An error occour while uploading zip to s3", exc_info=exc)
|
||||
raise HTTPException(
|
||||
status_code=500, detail="An error occour while uploading zip to s3."
|
||||
)
|
||||
else:
|
||||
response = FileResponse(
|
||||
file_path, filename=file_path.name, media_type="application/zip"
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "docling-serve"
|
||||
version = "1.0.1" # DO NOT EDIT, updated automatically
|
||||
version = "1.1.0" # DO NOT EDIT, updated automatically
|
||||
description = "Running Docling as a service"
|
||||
license = {text = "MIT"}
|
||||
authors = [
|
||||
@@ -35,8 +35,8 @@ classifiers = [
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"docling~=2.38",
|
||||
"docling-core>=2.32.0",
|
||||
"docling-jobkit[kfp,vlm]~=1.1",
|
||||
"docling-core>=2.44.1",
|
||||
"docling-jobkit[kfp,vlm]~=1.2",
|
||||
"fastapi[standard]~=0.115",
|
||||
"httpx~=0.28",
|
||||
"pydantic~=2.10",
|
||||
@@ -46,6 +46,7 @@ dependencies = [
|
||||
"uvicorn[standard]>=0.29.0,<1.0.0",
|
||||
"websockets~=14.0",
|
||||
"scalar-fastapi>=1.0.3",
|
||||
"docling-mcp>=1.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -128,7 +129,7 @@ torchvision = [
|
||||
{ index = "pytorch-cu126", group = "cu126" },
|
||||
{ index = "pytorch-cu128", group = "cu128" },
|
||||
]
|
||||
# docling-jobkit = { git = "https://github.com/docling-project/docling-jobkit/", rev = "refactor" }
|
||||
# docling-jobkit = { git = "https://github.com/docling-project/docling-jobkit/", rev = "main" }
|
||||
# docling-jobkit = { path = "../docling-jobkit", editable = true }
|
||||
|
||||
[[tool.uv.index]]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
@@ -8,6 +10,8 @@ from asgi_lifespan import LifespanManager
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from pytest_check import check
|
||||
|
||||
from docling_core.types.doc import DoclingDocument, PictureItem
|
||||
|
||||
from docling_serve.app import create_app
|
||||
|
||||
|
||||
@@ -153,3 +157,37 @@ async def test_convert_file(client: AsyncClient):
|
||||
data["document"]["doctags_content"],
|
||||
msg=f"DocTags document should contain '<doctag><page_header>'. Received: {safe_slice(data['document']['doctags_content'])}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_referenced_artifacts(client: AsyncClient):
|
||||
"""Test that paths in the zip file are relative to the zip file root."""
|
||||
|
||||
endpoint = "/v1/convert/file"
|
||||
options = {
|
||||
"to_formats": ["json"],
|
||||
"image_export_mode": "referenced",
|
||||
"target_type": "zip",
|
||||
"ocr": False,
|
||||
}
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
file_path = os.path.join(current_dir, "2206.01062v1.pdf")
|
||||
|
||||
files = {
|
||||
"files": ("2206.01062v1.pdf", open(file_path, "rb"), "application/pdf"),
|
||||
}
|
||||
|
||||
response = await client.post(endpoint, files=files, data=options)
|
||||
assert response.status_code == 200, "Response should be 200 OK"
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(response.content)) as zip_file:
|
||||
namelist = zip_file.namelist()
|
||||
for file in namelist:
|
||||
if file.endswith(".json"):
|
||||
doc = DoclingDocument.model_validate(json.loads(zip_file.read(file)))
|
||||
for item, _level in doc.iterate_items():
|
||||
if isinstance(item, PictureItem):
|
||||
assert item.image is not None
|
||||
print(f"{item.image.uri}=")
|
||||
assert str(item.image.uri) in namelist
|
||||
|
||||
Reference in New Issue
Block a user