Merge pull request #3520 from crazy-max/bin-image-github-builder

ci: use docker github builder to build bin image and binaries

.github/workflows/build.yml

@@ -346,129 +346,115 @@ jobs:
         with:
           sarif_file: ${{ env.DESTDIR }}/govulncheck.out
 
-  prepare-binaries:
-    runs-on: ubuntu-24.04
-    outputs:
-      matrix: ${{ steps.platforms.outputs.matrix }}
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v6
-      -
-        name: Create matrix
-        id: platforms
-        run: |
-          echo "matrix=$(docker buildx bake binaries-cross --print | jq -cr '.target."binaries-cross".platforms')" >>${GITHUB_OUTPUT}
-      -
-        name: Show matrix
-        run: |
-          echo ${{ steps.platforms.outputs.matrix }}
-
   binaries:
+    uses: docker/github-builder-experimental/.github/workflows/bake.yml@5876e8deef3c899c298ec80b07c43dd9e89d37f6
+    permissions:
+      contents: read # same as global permission
+      id-token: write # for signing attestation(s) with GitHub OIDC Token
+    with:
+      runner: amd64
+      artifact-name: buildx
+      artifact-upload: true
+      cache: true
+      cache-scope: bin-image
+      target: release
+      output: local
+      sbom: true
+      sign: ${{ github.event_name != 'pull_request' }}
+
+  binaries-finalize:
     runs-on: ubuntu-24.04
     needs:
-      - prepare-binaries
+      - binaries
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }}
     steps:
       -
-        name: Prepare
+        name: Download artifacts
-        run: |
+        uses: actions/download-artifact@v6
-          platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-      -
-        name: Checkout
-        uses: actions/checkout@v6
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
         with:
-          version: ${{ env.SETUP_BUILDX_VERSION }}
+          path: /tmp/buildx-output
-          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
+          name: ${{ needs.binaries.outputs.artifact-name }}
-          buildkitd-flags: --debug
       -
-        name: Build
+        name: Rename provenance and sbom
         run: |
-          make release
+          for pdir in /tmp/buildx-output/*/; do
-        env:
+            (
-          PLATFORMS: ${{ matrix.platform }}
+              cd "$pdir"
-          CACHE_FROM: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }}
+              binname=$(find . -name 'buildx-*')
-          CACHE_TO: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }},mode=max
+              filename=$(basename "${binname%.exe}")
+              mv "provenance.json" "${filename}.provenance.json"
+              mv "sbom-binaries.spdx.json" "${filename}.sbom.json"
+              find . -name 'sbom*.json' -exec rm {} \;
+              if [ -f "provenance.sigstore.json" ]; then
+                mv "provenance.sigstore.json" "${filename}.sigstore.json"
+              fi
+            )
+          done
+          mkdir -p "${{ env.DESTDIR }}"
+          mv /tmp/buildx-output/**/* "${{ env.DESTDIR }}/"
       -
-        name: Upload artifacts
+        name: Create checksums
+        working-directory: ${{ env.DESTDIR }}
+        run: |
+          sha256sum -b buildx-* > ./checksums.txt
+          sed -i '/darwin/d' ./checksums.txt
+          sha256sum -c --strict checksums.txt
+      -
+        name: List artifacts
+        working-directory: ${{ env.DESTDIR }}
+        run: |
+          tree -nh .
+      -
+        name: Check artifacts
+        working-directory: ${{ env.DESTDIR }}
+        run: |
+          find . -type f -exec file -e ascii -- {} +
+      -
+        name: Upload release binaries
         uses: actions/upload-artifact@v6
         with:
-          name: buildx-${{ env.PLATFORM_PAIR }}
+          name: release
           path: ${{ env.DESTDIR }}/*
           if-no-files-found: error
 
-  bin-image:
+  bin-image-prepare:
     runs-on: ubuntu-24.04
+    outputs:
+      repo-slug: ${{ env.REPO_SLUG }}
+    steps:
+      # FIXME: can't use env object in reusable workflow inputs: https://github.com/orgs/community/discussions/26671
+      - run: echo "Exposing env vars for reusable workflow"
+
+  bin-image:
+    if: ${{ github.repository == 'docker/buildx' }}
+    uses: docker/github-builder-experimental/.github/workflows/bake.yml@5876e8deef3c899c298ec80b07c43dd9e89d37f6
     needs:
+      - bin-image-prepare
       - test-integration
       - test-unit
-    if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }}
+    permissions:
-    steps:
+      contents: read # same as global permission
-      -
+      id-token: write # for signing attestation(s) with GitHub OIDC Token
-        name: Free disk space
+    with:
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+      runner: amd64
-        with:
+      target: image-cross
-          android: true
+      cache: true
-          dotnet: true
+      cache-scope: bin-image
-          haskell: true
+      output: image
-          large-packages: true
+      push: ${{ github.event_name != 'pull_request' }}
-          swap-storage: true
+      sbom: true
-      -
+      set-meta-labels: true
-        name: Checkout
+      meta-images: |
-        uses: actions/checkout@v6
+        ${{ needs.bin-image-prepare.outputs.repo-slug }}
-      -
+      meta-tags: |
-        name: Set up QEMU
+        type=ref,event=branch
-        uses: docker/setup-qemu-action@v3
+        type=ref,event=pr
-      -
+        type=semver,pattern={{version}}
-        name: Set up Docker Buildx
+      meta-bake-target: meta-helper
-        uses: docker/setup-buildx-action@v3
+    secrets:
-        with:
+      registry-auths: |
-          version: ${{ env.SETUP_BUILDX_VERSION }}
+        - registry: docker.io
-          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
-          buildkitd-flags: --debug
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.REPO_SLUG }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-          bake-target: meta-helper
-      -
-        name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
-        with:
           username: ${{ vars.DOCKERPUBLICBOT_USERNAME }}
           password: ${{ secrets.DOCKERPUBLICBOT_WRITE_PAT }}
-      -
-        name: Build and push image
-        uses: docker/bake-action@v6
-        with:
-          source: .
-          files: |
-            ./docker-bake.hcl
-            ${{ steps.meta.outputs.bake-file }}
-          targets: image-cross
-          push: ${{ github.event_name != 'pull_request' }}
-          sbom: true
-          set: |
-            *.cache-from=type=gha,scope=bin-image
-            *.cache-to=type=gha,scope=bin-image,mode=max
 
   scout:
     runs-on: ubuntu-24.04
@@ -509,29 +495,14 @@ jobs:
     needs:
       - test-integration
       - test-unit
-      - binaries
+      - binaries-finalize
     steps:
       -
-        name: Checkout
+        name: Download release binaries
-        uses: actions/checkout@v6
-      -
-        name: Download binaries
         uses: actions/download-artifact@v7
         with:
           path: ${{ env.DESTDIR }}
-          pattern: buildx-*
+          name: release
-          merge-multiple: true
-      -
-        name: Create checksums
-        run: ./hack/hash-files
-      -
-        name: List artifacts
-        run: |
-          tree -nh ${{ env.DESTDIR }}
-      -
-        name: Check artifacts
-        run: |
-          find ${{ env.DESTDIR }} -type f -exec file -e ascii -- {} +
       -
         name: GitHub Release
         if: startsWith(github.ref, 'refs/tags/v')

Makefile

@@ -30,10 +30,6 @@ install: binaries
 	mkdir -p ~/.docker/cli-plugins
 	install bin/build/buildx ~/.docker/cli-plugins/docker-buildx
 
-.PHONY: release
-release:
-	./hack/release
-
 .PHONY: validate-all
 validate-all: lint test validate-vendor validate-docs
 

hack/hash-files

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu -o pipefail
-
-: "${DESTDIR=./bin/release}"
-
-if [ ! -d "$DESTDIR" ]; then
-  exit 0
-fi
-
-# checksums
-(
-  cd ${DESTDIR}
-  sha256sum -b buildx-* > ./checksums.txt
-  sed -i '/darwin/d' ./checksums.txt
-  sha256sum -c --strict checksums.txt
-)

hack/release

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu -o pipefail
-
-: "${GITHUB_ACTIONS=}"
-: "${GITHUB_REPOSITORY=}"
-: "${GITHUB_RUN_ID=}"
-
-: "${BUILDX_CMD=docker buildx}"
-: "${DESTDIR=./bin/release}"
-: "${CACHE_FROM=}"
-: "${CACHE_TO=}"
-: "${PLATFORMS=}"
-
-if [ -n "$CACHE_FROM" ]; then
-  for cfrom in $CACHE_FROM; do
-    setFlags+=(--set "*.cache-from=$cfrom")
-  done
-fi
-if [ -n "$CACHE_TO" ]; then
-  for cto in $CACHE_TO; do
-    setFlags+=(--set "*.cache-to=$cto")
-  done
-fi
-if [ -n "$PLATFORMS" ]; then
-  setFlags+=(--set "*.platform=$PLATFORMS")
-fi
-if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then
-  prvattrs="mode=max"
-  if [ "$GITHUB_ACTIONS" = "true" ]; then
-    prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-  fi
-  setFlags+=(--set "*.attest=type=sbom")
-  setFlags+=(--set "*.attest=type=provenance,$prvattrs")
-fi
-
-output=$(mktemp -d -t buildx-output.XXXXXXXXXX)
-
-(
-  set -x
-  ${BUILDX_CMD} bake "${setFlags[@]}" --set "*.args.BUILDKIT_MULTI_PLATFORM=true" --set "*.output=$output" release
-)
-
-for pdir in "${output}"/*/; do
-  (
-    cd "$pdir"
-    binname=$(find . -name 'buildx-*')
-    filename=$(basename "${binname%.exe}")
-    mv "provenance.json" "${filename}.provenance.json"
-    mv "sbom-binaries.spdx.json" "${filename}.sbom.json"
-    find . -name 'sbom*.json' -exec rm {} \;
-  )
-done
-
-mkdir -p "$DESTDIR"
-mv "$output"/**/* "$DESTDIR/"
-rm -rf "$output"