From e1ea28e6a8f01f7817f44788e4e2f2b76c142469 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 21:05:57 -0800 Subject: [PATCH] ci: arm64 policy - serialise legs, one retry, never drop an existing arm64, ELF check before publish - max-parallel: 2 on the cac / cac-fpm matrices (8 concurrent QEMU legs failed on 2026-11-19) - ONE retry of a failed arm64 leg (90 s pause) in cac, cac-fpm, shared-httpd - assemble-multiarch-index.sh: --guard-existing-arm64 (fail instead of replacing a multi-arch tag by amd64-only; dispatch input allow_amd64_only overrides) and --verify-bin (ELF machine type of amd64 and arm64 before any tag is written) - tests: assemble-guard.test.sh, wiring tests; docs/arm64-publish-policy.md Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- .gitea/workflows/build-push.yaml | 97 ++++++++++++++++++++++-- docs/arm64-publish-policy.md | 8 ++ scripts/assemble-multiarch-index.sh | 52 ++++++++++++- scripts/tests/assemble-guard.test.sh | 79 +++++++++++++++++++ scripts/tests/multiarch-workflow.test.py | 49 +++++++++++- 5 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 docs/arm64-publish-policy.md create mode 100755 scripts/tests/assemble-guard.test.sh diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index 928f057..dafcac0 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -17,6 +17,11 @@ on: required: false default: '' type: string + allow_amd64_only: + description: 'true = let a build whose arm64 leg failed replace a tag that already has arm64 with an amd64-only one (default: fail instead; restore arm64 afterwards with the arm64-attach workflow).' + required: false + default: 'false' + type: string jobs: # Shell gate. Runs FIRST and costs seconds; the images below do not depend on @@ -79,6 +84,7 @@ jobs: scripts/index-platforms.sh \ scripts/assemble-multiarch-index.sh \ scripts/attach-arm64-index.sh \ + scripts/tests/assemble-guard.test.sh \ scripts/tests/attach-arm64-index.test.sh \ scripts/tests/verify-image-arch.test.sh \ scripts/tests/image-label-amd64.test.sh \ @@ -105,6 +111,7 @@ jobs: run: | ./scripts/tests/attach-arm64-index.test.sh ./scripts/tests/verify-image-arch.test.sh + ./scripts/tests/assemble-guard.test.sh # Static wiring of the multi-arch publish jobs (cac, cac-fpm, shared-httpd): # by-digest pushes, non-fatal arm64 leg, timeouts, exact tag scheme. @@ -162,6 +169,11 @@ jobs: # One PHP version failing (e.g. an EOL stream) must not cancel the # security builds of the others. fail-fast: false + # 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps + # (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded. + # Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether + # the runner honours it is unverified). + max-parallel: 2 matrix: phpver: ${{ fromJSON(needs.plan.outputs.cac) }} steps: @@ -301,13 +313,35 @@ jobs: EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/cac,push-by-digest=true,name-canonical=true,push=true + # ONE retry of a failed arm64 leg after a pause (network-step flakes). Same inputs. + - name: Wait before arm64 retry + if: steps.arm64.outcome == 'failure' + run: sleep 90 + + - name: Retry arm64 (push by digest) + id: arm64_retry + if: steps.arm64.outcome == 'failure' + continue-on-error: true + timeout-minutes: 45 + uses: docker/build-push-action@v6 + with: + platforms: linux/arm64 + build-args: | + PHPVER=${{ matrix.phpver }} + CACHE_BUST=${{ steps.bust.outputs.value }} + BASE_IMAGE=${{ steps.base.outputs.image }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} + EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} + outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/cac,push-by-digest=true,name-canonical=true,push=true + # Creates the index tag(s) from THIS run's digests only (a previous run's # arm64 is never re-attached). scripts/assemble-multiarch-index.sh. - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} - ARM64_OUTCOME: ${{ steps.arm64.outcome }} + ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} + ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} PHPVER: ${{ matrix.phpver }} run: | set -euo pipefail @@ -317,6 +351,7 @@ jobs: scripts/assemble-multiarch-index.sh \ --amd64 "$AMD64_DIGEST" \ --arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \ + --guard-existing-arm64 --verify-bin /usr/sbin/httpd --verify-bin /usr/bin/php \ --repo repo.anhonesthost.net/cloud-hosting-platform/cac \ "${tags[@]}" @@ -329,6 +364,11 @@ jobs: # One PHP version failing (e.g. an EOL stream) must not cancel the # security builds of the others. fail-fast: false + # 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps + # (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded. + # Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether + # the runner honours it is unverified). + max-parallel: 2 matrix: phpver: ${{ fromJSON(needs.plan.outputs.cac_fpm) }} steps: @@ -470,13 +510,36 @@ jobs: EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/cac-fpm,push-by-digest=true,name-canonical=true,push=true + # ONE retry of a failed arm64 leg after a pause (network-step flakes). Same inputs. + - name: Wait before arm64 retry + if: steps.arm64.outcome == 'failure' + run: sleep 90 + + - name: Retry arm64 (push by digest) + id: arm64_retry + if: steps.arm64.outcome == 'failure' + continue-on-error: true + timeout-minutes: 45 + uses: docker/build-push-action@v6 + with: + file: ./Dockerfile.fpm + platforms: linux/arm64 + build-args: | + PHPVER=${{ matrix.phpver }} + CACHE_BUST=${{ steps.bust.outputs.value }} + BASE_IMAGE=${{ steps.base.outputs.image }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} + EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} + outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/cac-fpm,push-by-digest=true,name-canonical=true,push=true + # Creates the index tag(s) from THIS run's digests only (a previous run's # arm64 is never re-attached). scripts/assemble-multiarch-index.sh. - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} - ARM64_OUTCOME: ${{ steps.arm64.outcome }} + ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} + ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} PHPVER: ${{ matrix.phpver }} run: | set -euo pipefail @@ -486,6 +549,7 @@ jobs: scripts/assemble-multiarch-index.sh \ --amd64 "$AMD64_DIGEST" \ --arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \ + --guard-existing-arm64 --verify-bin /usr/sbin/php-fpm --verify-bin /usr/bin/php \ --repo repo.anhonesthost.net/cloud-hosting-platform/cac-fpm \ "${tags[@]}" @@ -912,18 +976,39 @@ jobs: PKG_CACHE=${{ steps.pkgcache.outputs.url }} outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/shared-httpd,push-by-digest=true,name-canonical=true,push=true + # ONE retry of a failed arm64 leg after a pause (network-step flakes). Same inputs. + - name: Wait before arm64 retry + if: steps.arm64.outcome == 'failure' + run: sleep 90 + + - name: Retry arm64 (push by digest) + id: arm64_retry + if: steps.arm64.outcome == 'failure' + continue-on-error: true + timeout-minutes: 25 + uses: docker/build-push-action@v6 + with: + file: ./Dockerfile.shared-httpd + platforms: linux/arm64 + build-args: | + BASE_IMAGE=${{ steps.base.outputs.image }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} + outputs: type=image,name=repo.anhonesthost.net/cloud-hosting-platform/shared-httpd,push-by-digest=true,name-canonical=true,push=true + # Creates the index tag(s) from THIS run's digests only (a previous run's # arm64 is never re-attached). scripts/assemble-multiarch-index.sh. - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} - ARM64_OUTCOME: ${{ steps.arm64.outcome }} + ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} + ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} run: | set -euo pipefail scripts/assemble-multiarch-index.sh \ --amd64 "$AMD64_DIGEST" \ --arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \ + --guard-existing-arm64 --verify-bin /usr/sbin/httpd \ --repo repo.anhonesthost.net/cloud-hosting-platform/shared-httpd \ --tag latest diff --git a/docs/arm64-publish-policy.md b/docs/arm64-publish-policy.md new file mode 100644 index 0000000..b239843 --- /dev/null +++ b/docs/arm64-publish-policy.md @@ -0,0 +1,8 @@ +# arm64 in build-push.yaml (cac, cac-fpm, shared-httpd) + +The arm64 leg is best effort (QEMU on the x86 runner) but never silently lost: + +- Matrix families run at most 2 legs at once (`max-parallel: 2`); 2026-11-19 showed that 8 concurrent QEMU legs fail on network steps. +- A failed arm64 leg is retried ONCE after 90 s (`Retry arm64`). Assemble uses the first success. +- Before publishing, `scripts/assemble-multiarch-index.sh --verify-bin` checks ELF machine types (amd64 = x86-64, arm64 = AArch64). A wrong arm64 counts as a failed leg; a wrong amd64 aborts. +- `--guard-existing-arm64`: if the build ends without arm64 and the current tag has an arm64 entry, the assemble step FAILS and publishes nothing (the amd64 digest is in the error). Options: re-dispatch `only=`, or dispatch with `allow_amd64_only=true` (security rebuilds) and then restore arm64 with the `arm64-attach` workflow. Push-triggered runs have no input, so the guard is always on there. diff --git a/scripts/assemble-multiarch-index.sh b/scripts/assemble-multiarch-index.sh index 8da295c..d43b516 100755 --- a/scripts/assemble-multiarch-index.sh +++ b/scripts/assemble-multiarch-index.sh @@ -8,7 +8,8 @@ # --amd64 sha256: \ # --arm64 sha256: --arm64-outcome success|failure|cancelled|skipped \ # --repo [--repo ...] \ -# --tag [--tag ...] [--builder ] +# --tag [--tag ...] [--builder ] \ +# [--verify-bin PATH ...] [--guard-existing-arm64] # # Rules (all enforced here so the workflow stays thin and these are testable): # * amd64 is mandatory. A missing/malformed amd64 digest, or an amd64 digest @@ -19,16 +20,25 @@ # published amd64-only (warning, exit 0). The arm64 digest must come from # THIS run's step output - the workflow never reads it from a registry tag, # so a previous build's arm64 can never be re-attached to a newer amd64. +# * --verify-bin (repeatable): before anything is published, the amd64 entry must hold x86-64 +# and (if it would be attached) the arm64 entry AArch64 binaries at those paths +# (scripts/verify-image-arch.py, nothing is executed). Wrong/unverifiable amd64 = exit 1, +# nothing published. Wrong/unverifiable arm64 = treated as a failed arm64 leg. +# * --guard-existing-arm64: if the result would be amd64-only and a CURRENT tag already has a +# linux/arm64 entry, exit 1 and publish nothing (never silently drop arm64), unless +# env ALLOW_AMD64_ONLY=true. An unreadable tag fails closed; a missing tag is fine. # * every --tag is created in every --repo from @ sources in # that same repo (both digests were pushed to every --repo by the build). # -# Env: IMAGETOOLS overrides the command (default: "docker buildx imagetools"); +# Env: ALLOW_AMD64_ONLY=true lets the guard pass; VERIFY_ARCH overrides the verifier command +# (tests use a stub); IMAGETOOLS overrides the command (default: "docker buildx imagetools"); # the test suite points it at a stub. set -euo pipefail IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools} amd64=""; arm64=""; outcome="skipped"; builder="" -repos=(); tags=() +repos=(); tags=(); vbins=(); guard=0 +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) die() { echo "assemble-multiarch-index: $*" >&2; exit 1; } warn() { echo "::warning::assemble-multiarch-index: $*"; } @@ -41,6 +51,8 @@ while [ $# -gt 0 ]; do --repo) repos+=("${2-}"); shift 2 ;; --tag) tags+=("${2-}"); shift 2 ;; --builder) builder=${2-}; shift 2 ;; + --verify-bin) vbins+=("${2-}"); shift 2 ;; + --guard-existing-arm64) guard=1; shift ;; *) die "unknown argument: $1" ;; esac done @@ -80,6 +92,40 @@ else done fi +# ELF machine-type check (labels alone do not prove the binaries are of that architecture). +if [ ${#vbins[@]} -gt 0 ]; then + # shellcheck disable=SC2206 + verify=(${VERIFY_ARCH:-python3 -I "$HERE/verify-image-arch.py"}) + vargs=(); for b in "${vbins[@]}"; do vargs+=(--bin "$b"); done + for r in "${repos[@]}"; do + "${verify[@]}" --image "$r@$amd64" --platform linux/amd64 "${vargs[@]}" \ + || die "$r@$amd64 failed the x86-64 ELF check; refusing to publish" + if [ "$use_arm64" = 1 ]; then + if ! "${verify[@]}" --image "$r@$arm64" --platform linux/arm64 "${vargs[@]}"; then + warn "$r@$arm64 failed the AArch64 ELF check: publishing amd64-only" + use_arm64=0; break + fi + fi + done +fi + +# Never silently replace a multi-arch tag by an amd64-only one. +if [ "$guard" = 1 ] && [ "$use_arm64" = 0 ] && [ "${ALLOW_AMD64_ONLY:-false}" != true ]; then + for r in "${repos[@]}"; do for t in "${tags[@]}"; do + rc=0; info=$(IMAGETOOLS=$IMAGETOOLS "$HERE/index-platforms.sh" "$r:$t") || rc=$? + case "$rc" in + 0) ;; + 3) info="not found" ;; + *) echo "::error::assemble-multiarch-index: cannot tell whether $r:$t is multi-arch; refusing to overwrite it (fail closed)" >&2; exit 1 ;; + esac + case "$info" in + *linux/arm64*) + echo "::error::assemble-multiarch-index: $r:$t has an arm64 entry but this build has none (arm64 outcome '$outcome'); publishing amd64-only would drop it. Nothing published. amd64 digest of this build: $amd64. Fix: re-dispatch only=, or re-run with allow_amd64_only=true and then restore arm64 with the arm64-attach workflow." >&2 + exit 1 ;; + esac + done; done +fi + for r in "${repos[@]}"; do targs=(); for t in "${tags[@]}"; do targs+=(-t "$r:$t"); done srcs=("$r@$amd64"); [ "$use_arm64" = 1 ] && srcs+=("$r@$arm64") diff --git a/scripts/tests/assemble-guard.test.sh b/scripts/tests/assemble-guard.test.sh new file mode 100755 index 0000000..8bfbae0 --- /dev/null +++ b/scripts/tests/assemble-guard.test.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +## assemble-guard.test.sh - --guard-existing-arm64 and --verify-bin of scripts/assemble-multiarch-index.sh +## against a stub IMAGETOOLS (registry state in files) and a stub ELF verifier. +set -euo pipefail +HERE=$(cd "$(dirname "$0")/.." && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +fail=0 +chk() { if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got '$2' want '$3'"; fail=1; fi; } +d() { local c=$1; printf 'sha256:'; printf "$c%.0s" {1..64}; } +A=$(d a); B=$(d b); R=reg/x +cat >"$T/it" <<'STUB' +#!/usr/bin/env bash +# create -> logged. inspect --raw REF -> $FIX/ (err_ prefix = failure text). +# inspect REF --format .. -> architecture from $ARCHES (ref=arch lines). +if [ "$1" = create ]; then echo "create $*" >>"$LOG"; exit 0; fi +if [ "$2" = --raw ]; then + k=$(printf '%s' "$3" | tr '/:@' '___') + if [ -f "$FIX/err_$k" ]; then cat "$FIX/err_$k" >&2; exit 1; fi + [ -f "$FIX/$k" ] || { echo "ERROR: $3: not found" >&2; exit 1; } + cat "$FIX/$k"; exit 0 +fi +a=$(grep -F -m1 "$2=" "$ARCHES" | cut -d= -f2); [ -n "$a" ] || exit 1 +printf '{"architecture":"%s"}' "$a" +STUB +cat >"$T/ver" <<'STUB' +#!/usr/bin/env bash +echo "verify $*" >>"$LOG" +case "$*" in *linux/arm64*) [ "${FAIL_ARM:-0}" = 1 ] && exit 1 ;; *linux/amd64*) [ "${FAIL_AMD:-0}" = 1 ] && exit 1 ;; esac +exit 0 +STUB +chmod +x "$T/it" "$T/ver" +export IMAGETOOLS="$T/it" VERIFY_ARCH="$T/ver" FIX=$T LOG=$T/log ARCHES=$T/arches +printf '%s\n' "$R@$A=amd64" "$R@$B=arm64" >"$ARCHES" +w() { printf '%s' "$2" >"$T/$(printf '%s' "$1" | tr '/:@' '___')"; } +MULTI='{"manifests":[{"digest":"x","platform":{"os":"linux","architecture":"amd64"}},{"digest":"y","platform":{"os":"linux","architecture":"arm64"}}]}' +AMDONLY='{"manifests":[{"digest":"x","platform":{"os":"linux","architecture":"amd64"}}]}' +run() { : >"$LOG"; "$HERE/assemble-multiarch-index.sh" "$@" >"$T/out" 2>"$T/err"; echo $?; } +creates() { grep -c '^create' "$LOG" || true; } +base=(--amd64 "$A" --repo "$R" --tag php82) + +w $R:php82 "$MULTI" +chk "amd64-only result over a multi-arch tag, guard on -> 1" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1 +chk " nothing published" "$(creates)" 0 +chk " error names the amd64 digest" "$(grep -c "$A" "$T/err")" 1 +chk "arm64 leg skipped, guard on -> 1" "$(run "${base[@]}" --guard-existing-arm64)" 1 +chk "ALLOW_AMD64_ONLY=true overrides -> 0" "$(ALLOW_AMD64_ONLY=true run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0 +chk " amd64-only index created" "$(creates)" 1 +chk "ALLOW_AMD64_ONLY=false does not override" "$(ALLOW_AMD64_ONLY=false run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1 +chk "without --guard-existing-arm64 old behaviour (publish) -> 0" "$(run "${base[@]}" --arm64-outcome failure)" 0 +chk "arm64 present in this build: guard irrelevant -> 0" "$(run "${base[@]}" --arm64 "$B" --arm64-outcome success --guard-existing-arm64)" 0 +chk " 2 sources" "$(grep -c "$R@$A $R@$B\$" "$LOG")" 1 +w $R:php82 "$AMDONLY" +chk "existing tag amd64-only, guard on -> 0" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0 +rm -f "$T/reg_x_php82" +chk "tag not found (first publish) -> 0" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0 +printf 'ERROR: denied\n' >"$T/err_reg_x_php82" +chk "tag unreadable -> fail closed 1" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1 +chk " nothing published" "$(creates)" 0 +rm -f "$T/err_reg_x_php82" +w $R:php82 "$AMDONLY"; w $R:latest "$MULTI" +chk "second tag multi-arch is caught too -> 1" "$(run --amd64 "$A" --repo $R --tag php82 --tag latest --arm64-outcome failure --guard-existing-arm64)" 1 +chk " nothing published" "$(creates)" 0 + +# ---- ELF verification +w $R:php82 "$AMDONLY" +vb=(--verify-bin /usr/sbin/httpd --verify-bin /usr/bin/php) +chk "both verify -> 0, both platforms checked" "$(run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}")" 0 +chk " amd64 and arm64 verified with both bins" "$(grep -c "^verify --image $R@$A --platform linux/amd64 --bin /usr/sbin/httpd --bin /usr/bin/php\$\|^verify --image $R@$B --platform linux/arm64 --bin /usr/sbin/httpd --bin /usr/bin/php\$" "$LOG")" 2 +chk " arm64 index created" "$(grep -c "$R@$A $R@$B\$" "$LOG")" 1 +chk "wrong amd64 -> refuse 1" "$(FAIL_AMD=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}")" 1 +chk " nothing published" "$(creates)" 0 +chk "wrong arm64, tag amd64-only -> amd64-only publish 0" "$(FAIL_ARM=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}" --guard-existing-arm64)" 0 +chk " no arm64 digest in create" "$(grep -c "$B" <(grep '^create' "$LOG") || true)" 0 +w $R:php82 "$MULTI" +chk "wrong arm64 over multi-arch tag, guard -> 1 (falls into guard)" "$(FAIL_ARM=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}" --guard-existing-arm64)" 1 +chk " nothing published" "$(creates)" 0 +chk "arm64 failed: arm64 not verified at all" "$(run "${base[@]}" --arm64-outcome failure "${vb[@]}" >/dev/null; grep -c 'linux/arm64' "$LOG" || true)" 0 + +[ "$fail" -eq 0 ] && echo "ALL PASS" || { echo FAILURES; exit 1; } diff --git a/scripts/tests/multiarch-workflow.test.py b/scripts/tests/multiarch-workflow.test.py index 2289cfa..acaf02f 100755 --- a/scripts/tests/multiarch-workflow.test.py +++ b/scripts/tests/multiarch-workflow.test.py @@ -124,12 +124,57 @@ class Wiring(unittest.TestCase): for j, (repo, _, _) in SLICE.items(): st = step(self.jobs[j], 'Assemble') self.assertIn('${{ steps.amd64.outputs.digest }}', st, j) - self.assertIn('${{ steps.arm64.outputs.digest }}', st, j) - self.assertIn('${{ steps.arm64.outcome }}', st, j) + self.assertIn('${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}', st, j) + self.assertIn("steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success'", st, j) self.assertIn('scripts/assemble-multiarch-index.sh', st, j) self.assertIn('--repo %s/%s' % (REG, repo), st, j) self.assertEqual(st.count('--repo '), 1, j) + def test_one_retry_of_a_failed_arm64_leg(self): + for j in SLICE: + job = self.jobs[j] + names = [n for n, _ in steps(job)] + i = [k for k, n in enumerate(names) if n.startswith('Build arm64')][0] + self.assertTrue(names[i + 1].startswith('Wait before arm64 retry'), j) + self.assertTrue(names[i + 2].startswith('Retry arm64'), j) + self.assertLess(i + 2, [k for k, n in enumerate(names) if n.startswith('Assemble')][0], j) + first, retry = step(job, 'Build arm64'), step(job, 'Retry arm64') + self.assertIn("if: steps.arm64.outcome == 'failure'", step(job, 'Wait before'), j) + self.assertIn("if: steps.arm64.outcome == 'failure'", retry, j) + self.assertIn('id: arm64_retry', retry, j) + self.assertIn('continue-on-error: true', retry, j) + self.assertEqual(len(re.findall(r'Retry arm64', job)), 1, j) # exactly one retry + for key in ('timeout-minutes', 'platforms', 'file', 'outputs'): + a = re.search(r'(?m)^\s+%s: .*$' % key, first) + b = re.search(r'(?m)^\s+%s: .*$' % key, retry) + self.assertEqual(a and a.group(0), b and b.group(0), (j, key)) + ba = lambda x: re.search(r'build-args: \|\n((?: .*\n)+)', x).group(1) + self.assertEqual(ba(first), ba(retry), j) + + def test_matrix_families_cap_parallel_qemu_legs(self): + for j, (_, _, matrix) in SLICE.items(): + if matrix: + self.assertRegex(self.jobs[j], r'(?m)^ max-parallel: [12]$', j) + + def test_assemble_guard_verify_and_override_wiring(self): + want = {'Build-and-Push': ['/usr/sbin/httpd', '/usr/bin/php'], + 'Build-FPM-Images': ['/usr/sbin/php-fpm', '/usr/bin/php'], + 'Build-Shared-httpd': ['/usr/sbin/httpd']} + for j, bins in want.items(): + st = step(self.jobs[j], 'Assemble') + self.assertIn('--guard-existing-arm64', st, j) + self.assertEqual(re.findall(r'--verify-bin (\S+)', st), bins, j) + self.assertIn('ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }}', st, j) + run = re.search(r'run: \|\n((?: .*\n?)+)', st).group(1) + self.assertNotIn('${{', run, j) + + def test_allow_amd64_only_input_is_a_string_default_false(self): + head = TEXT.split('jobs:')[0] + m = re.search(r'(?m)^ allow_amd64_only:\n((?: .*\n)+)', head) + self.assertIsNotNone(m) + self.assertIn("default: 'false'", m.group(1)) + self.assertIn('type: string', m.group(1)) + def test_tag_scheme_unchanged(self): """Run the assemble step's shell with a recording stub: exact tags per version.""" for j, (repo, _, matrix) in SLICE.items():