Enable concurrent running of the image generation jobs (#4)

Group GPU image generate for GHCR and QUAY in one job,
and similarly group CPU only image generation for GHCR
and QUAY in another job. These two jobs can now run
concurrently.

By grouping the jobs for the same image type, it can
use the docker layer cache to run the second job faster.
If we separately build these images in four diffent and
concurrent jobs, they won't be able to use the cache.

If this still takes significantly long, we probably
have to generate image in it's own job, so that they can run
concurrently, without leveraging cache of docker layers.

Also added make file target to lint the github workflows.

Signed-off-by: Anil Vishnoi <[email protected]>

.github/workflows/images.yml

@@ -13,9 +13,13 @@ env:
   QUAY_DOCLING_SERVE_CPU_IMAGE_NAME: ds4sd/docling-serve-cpu
   QUAY_DOCLING_SERVE_GPU_IMAGE_NAME: ds4sd/docling-serve
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build_and_publish_ui_image:
-    name: Push docling-serve container images to GHCR and QUAY
+  build_and_publish_cpu_images:
+    name: Push docling-serve "CPU only" container image to GHCR and QUAY
     runs-on: ubuntu-latest
     environment: registry-creds
     permissions:
@@ -53,31 +57,13 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-buildx-
 
-      - name: Extract metadata (tags, labels) for docling-serve/cpu ghcr image
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) ghcr image
         id: ghcr_serve_cpu_meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_CPU_IMAGE_NAME }}
 
-      - name: Extract metadata (tags, labels) for docling-serve/gpu ghcr image
-        id: ghcr_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docling-serve/cpu quay image
-        id: quay_serve_cpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
-
-      - name: Extract metadata (tags, labels) for docking-serve/gpu quay image
-        id: quay_serve_gpu_meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}
-
-      - name: Build and push docling-serve/cpu image to ghcr.io
+      - name: Build and push docling-serve-cpu image to ghcr.io
         id: push-serve-cpu-ghcr
         uses: docker/build-push-action@v5
         with:
@@ -99,7 +85,73 @@ jobs:
           subject-digest: ${{ steps.push-serve-cpu-ghcr.outputs.digest }}
           push-to-registry: true
 
-      - name: Build and push docling-serve/gpu image to ghcr.io
+      - name: Extract metadata (tags, labels) for docling-serve (CPU only) quay image
+        id: quay_serve_cpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_CPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve-cpu image to quay.io
+        id: push-serve-cpu-quay
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
+          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
+          platforms: linux/amd64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: Containerfile
+          build-args: |
+            --build-arg CPU_ONLY=true
+
+  build_and_publish_gpu_images:
+    name: Push docling-serve (with GPU support) container image to GHCR and QUAY
+    runs-on: ubuntu-latest
+    environment: registry-creds
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to the GHCR container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GHCR_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log in to the Quay container image registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.QUAY_REGISTRY }}
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) ghcr image
+        id: ghcr_serve_gpu_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_DOCLING_SERVE_GPU_IMAGE_NAME }}
+
+      - name: Build and push docling-serve (GPU) image to ghcr.io
         id: push-serve-gpu-ghcr
         uses: docker/build-push-action@v5
         with:
@@ -121,22 +173,13 @@ jobs:
           subject-digest: ${{ steps.push-serve-gpu-ghcr.outputs.digest }}
           push-to-registry: true
 
-      - name: Build and push docling-serve/cpu image to quay.io
-        id: push-serve-cpu-quay
-        uses: docker/build-push-action@v5
+      - name: Extract metadata (tags, labels) for docling-serve (GPU) quay image
+        id: quay_serve_gpu_meta
+        uses: docker/metadata-action@v5
         with:
-          context: .
-          push: true
-          tags: ${{ steps.quay_serve_cpu_meta.outputs.tags }}
-          labels: ${{ steps.quay_serve_cpu_meta.outputs.labels }}
-          platforms: linux/amd64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          file: Containerfile
-          build-args: |
-            --build-arg CPU_ONLY=true
+          images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_DOCLING_SERVE_GPU_IMAGE_NAME }}
 
-      - name: Build and push docling-serve/gpu image to quay.io
+      - name: Build and push docling-serve (GPU) image to quay.io
         id: push-serve-gpu-quay
         uses: docker/build-push-action@v5
         with:
@@ -149,4 +192,4 @@ jobs:
           cache-to: type=gha,mode=max
           file: Containerfile
           build-args: |
-            --build-arg CPU_ONLY=false
+            --build-arg CPU_ONLY=false

.gitignore

@@ -438,3 +438,6 @@ pip-selfcheck.json
 # Docs
 # docs/**/*.png
 # docs/**/*.svg
+
+# Makefile
+.action-lint

Makefile

@@ -9,21 +9,45 @@ help:
 ifeq ($(NOISY_BUILD),)
     ECHO_PREFIX=@
     CMD_PREFIX=@
+    PIPE_DEV_NULL=> /dev/null 2> /dev/null
 else
     ECHO_PREFIX=@\#
-    CMD_PREFIX=    PIPE_DEV_NULL=
+    CMD_PREFIX=
+    PIPE_DEV_NULL=
 endif
 
 TAG=$(shell git rev-parse HEAD)
 
+lint:
+	$(CMD_PREFIX) touch .action-lint
+
+.PHONY: docling-serve-cpu-image
 docling-serve-cpu-image: Containerfile ## Build docling-serve "cpu only" continaer image
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve CPU ONLY]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=true -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve-cpu:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) ghcr.io/ds4sd/docling-serve-cpu:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve-cpu:$(TAG) quay.io/ds4sd/docling-serve-cpu:main
 
+.PHONY: docling-serve-gpu-image
 docling-serve-gpu-image: Containerfile ## Build docling-serve continaer image with GPU support
 	$(ECHO_PREFIX) printf "  %-12s Containerfile\n" "[docling-serve with GPU]"
 	$(CMD_PREFIX) docker build --build-arg CPU_ONLY=false -f Containerfile --platform linux/amd64 -t ghcr.io/ds4sd/docling-serve:$(TAG) .
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) ghcr.io/ds4sd/docling-serve:main
 	$(CMD_PREFIX) docker tag ghcr.io/ds4sd/docling-serve:$(TAG) quay.io/ds4sd/docling-serve:main
+
+.PHONY: action-lint
+action-lint: .action-lint ##      Lint GitHub Action workflows
+.action-lint: $(shell find .github -type f) | lint
+	$(ECHO_PREFIX) printf "  %-12s .github/...\n" "[ACTION LINT]"
+	$(CMD_PREFIX) if ! which actionlint $(PIPE_DEV_NULL) ; then \
+		echo "Please install actionlint." ; \
+		echo "go install github.com/rhysd/actionlint/cmd/actionlint@latest" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) if ! which shellcheck $(PIPE_DEV_NULL) ; then \
+		echo "Please install shellcheck." ; \
+		echo "https://github.com/koalaman/shellcheck#user-content-installing" ; \
+		exit 1 ; \
+	fi
+	$(CMD_PREFIX) actionlint -color
+	$(CMD_PREFIX) touch $@