Enable concurrent running of the image generation jobs (#4)

Group GPU image generate for GHCR and QUAY in one job, and similarly group CPU only image generation for GHCR and QUAY in another job. These two jobs can now run concurrently. By grouping the jobs for the same image type, it can use the docker layer cache to run the second job faster. If we separately build these images in four diffent and concurrent jobs, they won't be able to use the cache. If this still takes significantly long, we probably have to generate image in it's own job, so that they can run concurrently, without leveraging cache of docker layers. Also added make file target to lint the github workflows. Signed-off-by: Anil Vishnoi <vishnoianil@gmail.com>
2025-11-29 08:33:50 +00:00 · 2024-09-13 01:31:06 -07:00
parent 4dd517c96a
commit fc11dc2540
3 changed files with 109 additions and 39 deletions
--- a/.github/workflows/images.yml
+++ b/.github/workflows/images.yml
@@ -13,9 +13,13 @@ env:
  QUAY_DOCLING_SERVE_CPU_IMAGE_NAME: ds4sd/docling-serve-cpu
  QUAY_DOCLING_SERVE_GPU_IMAGE_NAME: ds4sd/docling-serve

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build_and_publish_ui_image:
-    name: Push docling-serve container images to GHCR and QUAY
+  build_and_publish_cpu_images:
+    name: Push docling-serve "CPU only" container image to GHCR and QUAY
    runs-on: ubuntu-latest
    environment: registry-creds
    permissions:
@@ -53,31 +57,13 @@ jobs:
          restore-keys: |
            ${{ runner.os }}-buildx-

-      - name: Extract metadata (tags, labels) for docling-serve/cpu ghcr image
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) ghcr image
        id: ghcr_serve_cpu_meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_CPU_IMAGE_NAME }}

-      - name: Extract metadata (tags, labels) for docling-serve/gpu ghcr image
-        id: ghcr_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docling-serve/cpu quay image
-        id: quay_serve_cpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docking-serve/gpu quay image
-        id: quay_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Build and push docling-serve/cpu image to ghcr.io
+      - name: Build and push docling-serve-cpu image to ghcr.io
        id: push-serve-cpu-ghcr
        uses: docker/build-push-action@v5
        with:
@@ -99,7 +85,73 @@ jobs:
          subject-digest: ${{ steps.push-serve-cpu-ghcr.outputs.digest }}
          push-to-registry: true

-      - name: Build and push docling-serve/gpu image to ghcr.io
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) quay image
+        id: quay_serve_cpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve-cpu image to quay.io
+        id: push-serve-cpu-quay
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
+          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: Containerfile
+          build-args: |
+            --build-arg CPU_ONLY=true
+
+  build_and_publish_gpu_images:
+    name: Push docling-serve (with GPU support) container image to GHCR and QUAY
+    runs-on: ubuntu-latest
+    environment: registry-creds
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to the GHCR container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GHCR_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log in to the Quay container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.QUAY_REGISTRY }}
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) ghcr image
+        id: ghcr_serve_gpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve (GPU) image to ghcr.io
        id: push-serve-gpu-ghcr
        uses: docker/build-push-action@v5
        with:
@@ -121,22 +173,13 @@ jobs:
          subject-digest: ${{ steps.push-serve-gpu-ghcr.outputs.digest }}
          push-to-registry: true

-      - name: Build and push docling-serve/cpu image to quay.io
-        id: push-serve-cpu-quay
-        uses: docker/build-push-action@v5
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) quay image
+        id: quay_serve_gpu_meta
+        uses: docker/metadata-action@v5
        with:
-          context: .
-          push: true
-          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
-          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
-          platforms: linux/amd64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          file: Containerfile
-          build-args: |
-            --build-arg CPU_ONLY=true
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}

-      - name: Build and push docling-serve/gpu image to quay.io
+      - name: Build and push docling-serve (GPU) image to quay.io
        id: push-serve-gpu-quay
        uses: docker/build-push-action@v5
        with:
@@ -149,4 +192,4 @@ jobs:
          cache-to: type=gha,mode=max
          file: Containerfile
          build-args: |
-            --build-arg CPU_ONLY=false
+            --build-arg CPU_ONLY=false
--- a/.gitignore
+++ b/.gitignore
@@ -438,3 +438,6 @@ pip-selfcheck.json
 # Docs
 # docs/**/*.png
 # docs/**/*.svg
+
+# Makefile
+.action-lint
--- a/26
+++ b/26
@@ -9,21 +9,45 @@ help:
 ifeq ($(NOISY_BUILD),)
    ECHO_PREFIX=@
    CMD_PREFIX=@
+    PIPE_DEV_NULL=> /dev/null 2> /dev/null
 else
    ECHO_PREFIX=@\#
-    CMD_PREFIX=    PIPE_DEV_NULL=
+    CMD_PREFIX=
+    PIPE_DEV_NULL=
 endif

 TAG=$(shell git rev-parse HEAD)

+lint:
+	$(CMD_PREFIX) touch .action-lint
+
+.PHONY: docling-serve-cpu-image
 docling-serve-cpu-image: Containerfile ## Build docling-serve "cpu only" continaer image
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve CPU ONLY]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=true -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve-cpu:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) ghcr.io/ds4sd/docling-serve-cpu:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) quay.io/ds4sd/docling-serve-cpu:main

+.PHONY: docling-serve-gpu-image
 docling-serve-gpu-image: Containerfile ## Build docling-serve continaer image with GPU support
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve with GPU]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=false -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) ghcr.io/ds4sd/docling-serve:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) quay.io/ds4sd/docling-serve:main
+
+.PHONY: action-lint
+action-lint: .action-lint ##      Lint GitHub Action workflows
+.action-lint: $(shell find .github -type f) | lint
+	$(ECHO_PREFIX) printf "  %-12s .github/...\n" "[ACTION LINT]"
+	$(CMD_PREFIX) if ! which actionlint $(PIPE_DEV_NULL) ; then \
+		echo "Please install actionlint." ; \
+		echo "go install github.com/rhysd/actionlint/cmd/actionlint@latest" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) if ! which shellcheck $(PIPE_DEV_NULL) ; then \
+		echo "Please install shellcheck." ; \
+		echo "https://github.com/koalaman/shellcheck#user-content-installing" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) actionlint -color
+	$(CMD_PREFIX) touch $@