From fed6566755970ccd1e2694a02244a543f74b8d3b Mon Sep 17 00:00:00 2001 From: Jeff Larson Date: Sat, 4 Jul 2026 13:38:15 -0700 Subject: [PATCH] ci: native per-arch builders (buildx --append), retire QEMU amd64 on buildkit (node-3) + arm64 on buildkit-arm64 (arm64 Pi), native; buildx merges. Same client cert (shared buildkit-ca). QEMU arm64 Rust builds ran for hours. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4492550..2833056 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,11 +81,12 @@ jobs: # One image now: the server builds the UI and embeds it (rust-embed), so there # is no separate watcher-ui image. Build context is the repo root. # - # Native arm64 on the self-hosted runner (watcher-runners), offloading to the - # in-cluster BuildKit — no QEMU, since the cluster is ARM-only. BuildKit enforces - # mTLS, so we point the remote driver at the client cert the runner already - # mounts (BUILDKIT_TLS_DIR) and verify against its servername; without the cert - # the gRPC handshake is rejected. The build also pushes a layer cache to GHA + # Native per-arch on the self-hosted runner (watcher-runners), offloading to the + # two in-cluster BuildKit builders — amd64 on `buildkit` (node-3), arm64 on + # `buildkit-arm64` (an arm64 Pi) — each building NATIVELY, no QEMU. BuildKit + # enforces mTLS, so we point the remote driver at the client cert the runner + # already mounts (BUILDKIT_TLS_DIR) and verify each endpoint against its + # servername; without the cert the gRPC handshake is rejected. The build also pushes a layer cache to GHA # (cache-from/to type=gha): paired with cargo-chef in the Dockerfile, the dep # compile becomes a portable layer any node can pull — unlike the per-node # BuildKit daemon cache, which a cold node can't reach. @@ -99,16 +100,29 @@ jobs: id-token: write # keyless cosign signing (OIDC token -> Sigstore Fulcio) steps: - uses: actions/checkout@v7 - - name: Connect to BuildKit (mTLS) + - name: Connect to BuildKit (mTLS, native per-arch) + # Two native builders share one CA (cluster: charts/actions/buildkit + + # values-arm64.yaml): amd64 on `buildkit` (node-3), arm64 on `buildkit-arm64` + # (an arm64 Pi). buildx --append joins them into one builder so each platform + # builds NATIVELY and buildx merges the manifest — no QEMU (arm64 Rust under + # QEMU ran for hours). The runner's existing client cert (BUILDKIT_TLS_DIR) + # authenticates to both; each endpoint verifies its own server SAN via servername. + env: + BUILDKIT_ARM64_HOST: tcp://buildkit-arm64.dev.svc.cluster.local:1234 + BUILDKIT_ARM64_TLS_SERVER_NAME: buildkit-arm64.dev.svc.cluster.local run: | for i in 1 2 3 4 5; do docker buildx rm watcher-remote >/dev/null 2>&1 || true if docker buildx create --name watcher-remote --use \ - --driver remote \ + --driver remote --platform linux/amd64 \ --driver-opt cacert="$BUILDKIT_TLS_DIR/ca.crt",cert="$BUILDKIT_TLS_DIR/tls.crt",key="$BUILDKIT_TLS_DIR/tls.key",servername="$BUILDKIT_TLS_SERVER_NAME" \ "$BUILDKIT_HOST" \ + && docker buildx create --name watcher-remote --append \ + --driver remote --platform linux/arm64 \ + --driver-opt cacert="$BUILDKIT_TLS_DIR/ca.crt",cert="$BUILDKIT_TLS_DIR/tls.crt",key="$BUILDKIT_TLS_DIR/tls.key",servername="$BUILDKIT_ARM64_TLS_SERVER_NAME" \ + "$BUILDKIT_ARM64_HOST" \ && timeout 90 docker buildx inspect --bootstrap; then - echo "connected to buildkit"; exit 0 + echo "connected to buildkit (amd64 + arm64, native)"; exit 0 fi echo "buildkit connect failed (attempt $i/5), retrying…"; sleep 10 done