ci: arm64 policy - serialise legs, one retry, never drop an existing arm64, ELF check before publish
- max-parallel: 2 on the cac / cac-fpm matrices (8 concurrent QEMU legs failed on 2026-11-19) - ONE retry of a failed arm64 leg (90 s pause) in cac, cac-fpm, shared-httpd - assemble-multiarch-index.sh: --guard-existing-arm64 (fail instead of replacing a multi-arch tag by amd64-only; dispatch input allow_amd64_only overrides) and --verify-bin (ELF machine type of amd64 and arm64 before any tag is written) - tests: assemble-guard.test.sh, wiring tests; docs/arm64-publish-policy.md Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
1 parent
655c88f10b
commit
e1ea28e6a8
5 files changed
+274
-11
No files matched your search
@@ -8,7 +8,8 @@
|
||||
# --amd64 sha256:<digest> \
|
||||
# --arm64 sha256:<digest> --arm64-outcome success|failure|cancelled|skipped \
|
||||
# --repo <registry/path> [--repo <other registry/path> ...] \
|
||||
# --tag <tag> [--tag <tag> ...] [--builder <buildx builder>]
|
||||
# --tag <tag> [--tag <tag> ...] [--builder <buildx builder>] \
|
||||
# [--verify-bin PATH ...] [--guard-existing-arm64]
|
||||
#
|
||||
# Rules (all enforced here so the workflow stays thin and these are testable):
|
||||
# * amd64 is mandatory. A missing/malformed amd64 digest, or an amd64 digest
|
||||
@@ -19,16 +20,25 @@
|
||||
# published amd64-only (warning, exit 0). The arm64 digest must come from
|
||||
# THIS run's step output - the workflow never reads it from a registry tag,
|
||||
# so a previous build's arm64 can never be re-attached to a newer amd64.
|
||||
# * --verify-bin (repeatable): before anything is published, the amd64 entry must hold x86-64
|
||||
# and (if it would be attached) the arm64 entry AArch64 binaries at those paths
|
||||
# (scripts/verify-image-arch.py, nothing is executed). Wrong/unverifiable amd64 = exit 1,
|
||||
# nothing published. Wrong/unverifiable arm64 = treated as a failed arm64 leg.
|
||||
# * --guard-existing-arm64: if the result would be amd64-only and a CURRENT tag already has a
|
||||
# linux/arm64 entry, exit 1 and publish nothing (never silently drop arm64), unless
|
||||
# env ALLOW_AMD64_ONLY=true. An unreadable tag fails closed; a missing tag is fine.
|
||||
# * every --tag is created in every --repo from <repo>@<digest> sources in
|
||||
# that same repo (both digests were pushed to every --repo by the build).
|
||||
#
|
||||
# Env: IMAGETOOLS overrides the command (default: "docker buildx imagetools");
|
||||
# Env: ALLOW_AMD64_ONLY=true lets the guard pass; VERIFY_ARCH overrides the verifier command
|
||||
# (tests use a stub); IMAGETOOLS overrides the command (default: "docker buildx imagetools");
|
||||
# the test suite points it at a stub.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools}
|
||||
amd64=""; arm64=""; outcome="skipped"; builder=""
|
||||
repos=(); tags=()
|
||||
repos=(); tags=(); vbins=(); guard=0
|
||||
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
die() { echo "assemble-multiarch-index: $*" >&2; exit 1; }
|
||||
warn() { echo "::warning::assemble-multiarch-index: $*"; }
|
||||
@@ -41,6 +51,8 @@ while [ $# -gt 0 ]; do
|
||||
--repo) repos+=("${2-}"); shift 2 ;;
|
||||
--tag) tags+=("${2-}"); shift 2 ;;
|
||||
--builder) builder=${2-}; shift 2 ;;
|
||||
--verify-bin) vbins+=("${2-}"); shift 2 ;;
|
||||
--guard-existing-arm64) guard=1; shift ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
@@ -80,6 +92,40 @@ else
|
||||
done
|
||||
fi
|
||||
|
||||
# ELF machine-type check (labels alone do not prove the binaries are of that architecture).
|
||||
if [ ${#vbins[@]} -gt 0 ]; then
|
||||
# shellcheck disable=SC2206
|
||||
verify=(${VERIFY_ARCH:-python3 -I "$HERE/verify-image-arch.py"})
|
||||
vargs=(); for b in "${vbins[@]}"; do vargs+=(--bin "$b"); done
|
||||
for r in "${repos[@]}"; do
|
||||
"${verify[@]}" --image "$r@$amd64" --platform linux/amd64 "${vargs[@]}" \
|
||||
|| die "$r@$amd64 failed the x86-64 ELF check; refusing to publish"
|
||||
if [ "$use_arm64" = 1 ]; then
|
||||
if ! "${verify[@]}" --image "$r@$arm64" --platform linux/arm64 "${vargs[@]}"; then
|
||||
warn "$r@$arm64 failed the AArch64 ELF check: publishing amd64-only"
|
||||
use_arm64=0; break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Never silently replace a multi-arch tag by an amd64-only one.
|
||||
if [ "$guard" = 1 ] && [ "$use_arm64" = 0 ] && [ "${ALLOW_AMD64_ONLY:-false}" != true ]; then
|
||||
for r in "${repos[@]}"; do for t in "${tags[@]}"; do
|
||||
rc=0; info=$(IMAGETOOLS=$IMAGETOOLS "$HERE/index-platforms.sh" "$r:$t") || rc=$?
|
||||
case "$rc" in
|
||||
0) ;;
|
||||
3) info="not found" ;;
|
||||
*) echo "::error::assemble-multiarch-index: cannot tell whether $r:$t is multi-arch; refusing to overwrite it (fail closed)" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$info" in
|
||||
*linux/arm64*)
|
||||
echo "::error::assemble-multiarch-index: $r:$t has an arm64 entry but this build has none (arm64 outcome '$outcome'); publishing amd64-only would drop it. Nothing published. amd64 digest of this build: $amd64. Fix: re-dispatch only=<leg>, or re-run with allow_amd64_only=true and then restore arm64 with the arm64-attach workflow." >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
done; done
|
||||
fi
|
||||
|
||||
for r in "${repos[@]}"; do
|
||||
targs=(); for t in "${tags[@]}"; do targs+=(-t "$r:$t"); done
|
||||
srcs=("$r@$amd64"); [ "$use_arm64" = 1 ] && srcs+=("$r@$arm64")
|
||||
|
||||
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
## assemble-guard.test.sh - --guard-existing-arm64 and --verify-bin of scripts/assemble-multiarch-index.sh
|
||||
## against a stub IMAGETOOLS (registry state in files) and a stub ELF verifier.
|
||||
set -euo pipefail
|
||||
HERE=$(cd "$(dirname "$0")/.." && pwd)
|
||||
T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
|
||||
fail=0
|
||||
chk() { if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got '$2' want '$3'"; fail=1; fi; }
|
||||
d() { local c=$1; printf 'sha256:'; printf "$c%.0s" {1..64}; }
|
||||
A=$(d a); B=$(d b); R=reg/x
|
||||
cat >"$T/it" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
# create -> logged. inspect --raw REF -> $FIX/<ref with /:@ as _> (err_ prefix = failure text).
|
||||
# inspect REF --format .. -> architecture from $ARCHES (ref=arch lines).
|
||||
if [ "$1" = create ]; then echo "create $*" >>"$LOG"; exit 0; fi
|
||||
if [ "$2" = --raw ]; then
|
||||
k=$(printf '%s' "$3" | tr '/:@' '___')
|
||||
if [ -f "$FIX/err_$k" ]; then cat "$FIX/err_$k" >&2; exit 1; fi
|
||||
[ -f "$FIX/$k" ] || { echo "ERROR: $3: not found" >&2; exit 1; }
|
||||
cat "$FIX/$k"; exit 0
|
||||
fi
|
||||
a=$(grep -F -m1 "$2=" "$ARCHES" | cut -d= -f2); [ -n "$a" ] || exit 1
|
||||
printf '{"architecture":"%s"}' "$a"
|
||||
STUB
|
||||
cat >"$T/ver" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
echo "verify $*" >>"$LOG"
|
||||
case "$*" in *linux/arm64*) [ "${FAIL_ARM:-0}" = 1 ] && exit 1 ;; *linux/amd64*) [ "${FAIL_AMD:-0}" = 1 ] && exit 1 ;; esac
|
||||
exit 0
|
||||
STUB
|
||||
chmod +x "$T/it" "$T/ver"
|
||||
export IMAGETOOLS="$T/it" VERIFY_ARCH="$T/ver" FIX=$T LOG=$T/log ARCHES=$T/arches
|
||||
printf '%s\n' "$R@$A=amd64" "$R@$B=arm64" >"$ARCHES"
|
||||
w() { printf '%s' "$2" >"$T/$(printf '%s' "$1" | tr '/:@' '___')"; }
|
||||
MULTI='{"manifests":[{"digest":"x","platform":{"os":"linux","architecture":"amd64"}},{"digest":"y","platform":{"os":"linux","architecture":"arm64"}}]}'
|
||||
AMDONLY='{"manifests":[{"digest":"x","platform":{"os":"linux","architecture":"amd64"}}]}'
|
||||
run() { : >"$LOG"; "$HERE/assemble-multiarch-index.sh" "$@" >"$T/out" 2>"$T/err"; echo $?; }
|
||||
creates() { grep -c '^create' "$LOG" || true; }
|
||||
base=(--amd64 "$A" --repo "$R" --tag php82)
|
||||
|
||||
w $R:php82 "$MULTI"
|
||||
chk "amd64-only result over a multi-arch tag, guard on -> 1" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1
|
||||
chk " nothing published" "$(creates)" 0
|
||||
chk " error names the amd64 digest" "$(grep -c "$A" "$T/err")" 1
|
||||
chk "arm64 leg skipped, guard on -> 1" "$(run "${base[@]}" --guard-existing-arm64)" 1
|
||||
chk "ALLOW_AMD64_ONLY=true overrides -> 0" "$(ALLOW_AMD64_ONLY=true run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0
|
||||
chk " amd64-only index created" "$(creates)" 1
|
||||
chk "ALLOW_AMD64_ONLY=false does not override" "$(ALLOW_AMD64_ONLY=false run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1
|
||||
chk "without --guard-existing-arm64 old behaviour (publish) -> 0" "$(run "${base[@]}" --arm64-outcome failure)" 0
|
||||
chk "arm64 present in this build: guard irrelevant -> 0" "$(run "${base[@]}" --arm64 "$B" --arm64-outcome success --guard-existing-arm64)" 0
|
||||
chk " 2 sources" "$(grep -c "$R@$A $R@$B\$" "$LOG")" 1
|
||||
w $R:php82 "$AMDONLY"
|
||||
chk "existing tag amd64-only, guard on -> 0" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0
|
||||
rm -f "$T/reg_x_php82"
|
||||
chk "tag not found (first publish) -> 0" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 0
|
||||
printf 'ERROR: denied\n' >"$T/err_reg_x_php82"
|
||||
chk "tag unreadable -> fail closed 1" "$(run "${base[@]}" --arm64-outcome failure --guard-existing-arm64)" 1
|
||||
chk " nothing published" "$(creates)" 0
|
||||
rm -f "$T/err_reg_x_php82"
|
||||
w $R:php82 "$AMDONLY"; w $R:latest "$MULTI"
|
||||
chk "second tag multi-arch is caught too -> 1" "$(run --amd64 "$A" --repo $R --tag php82 --tag latest --arm64-outcome failure --guard-existing-arm64)" 1
|
||||
chk " nothing published" "$(creates)" 0
|
||||
|
||||
# ---- ELF verification
|
||||
w $R:php82 "$AMDONLY"
|
||||
vb=(--verify-bin /usr/sbin/httpd --verify-bin /usr/bin/php)
|
||||
chk "both verify -> 0, both platforms checked" "$(run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}")" 0
|
||||
chk " amd64 and arm64 verified with both bins" "$(grep -c "^verify --image $R@$A --platform linux/amd64 --bin /usr/sbin/httpd --bin /usr/bin/php\$\|^verify --image $R@$B --platform linux/arm64 --bin /usr/sbin/httpd --bin /usr/bin/php\$" "$LOG")" 2
|
||||
chk " arm64 index created" "$(grep -c "$R@$A $R@$B\$" "$LOG")" 1
|
||||
chk "wrong amd64 -> refuse 1" "$(FAIL_AMD=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}")" 1
|
||||
chk " nothing published" "$(creates)" 0
|
||||
chk "wrong arm64, tag amd64-only -> amd64-only publish 0" "$(FAIL_ARM=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}" --guard-existing-arm64)" 0
|
||||
chk " no arm64 digest in create" "$(grep -c "$B" <(grep '^create' "$LOG") || true)" 0
|
||||
w $R:php82 "$MULTI"
|
||||
chk "wrong arm64 over multi-arch tag, guard -> 1 (falls into guard)" "$(FAIL_ARM=1 run "${base[@]}" --arm64 "$B" --arm64-outcome success "${vb[@]}" --guard-existing-arm64)" 1
|
||||
chk " nothing published" "$(creates)" 0
|
||||
chk "arm64 failed: arm64 not verified at all" "$(run "${base[@]}" --arm64-outcome failure "${vb[@]}" >/dev/null; grep -c 'linux/arm64' "$LOG" || true)" 0
|
||||
|
||||
[ "$fail" -eq 0 ] && echo "ALL PASS" || { echo FAILURES; exit 1; }
|
||||
@@ -124,12 +124,57 @@ class Wiring(unittest.TestCase):
|
||||
for j, (repo, _, _) in SLICE.items():
|
||||
st = step(self.jobs[j], 'Assemble')
|
||||
self.assertIn('${{ steps.amd64.outputs.digest }}', st, j)
|
||||
self.assertIn('${{ steps.arm64.outputs.digest }}', st, j)
|
||||
self.assertIn('${{ steps.arm64.outcome }}', st, j)
|
||||
self.assertIn('${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}', st, j)
|
||||
self.assertIn("steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success'", st, j)
|
||||
self.assertIn('scripts/assemble-multiarch-index.sh', st, j)
|
||||
self.assertIn('--repo %s/%s' % (REG, repo), st, j)
|
||||
self.assertEqual(st.count('--repo '), 1, j)
|
||||
|
||||
def test_one_retry_of_a_failed_arm64_leg(self):
|
||||
for j in SLICE:
|
||||
job = self.jobs[j]
|
||||
names = [n for n, _ in steps(job)]
|
||||
i = [k for k, n in enumerate(names) if n.startswith('Build arm64')][0]
|
||||
self.assertTrue(names[i + 1].startswith('Wait before arm64 retry'), j)
|
||||
self.assertTrue(names[i + 2].startswith('Retry arm64'), j)
|
||||
self.assertLess(i + 2, [k for k, n in enumerate(names) if n.startswith('Assemble')][0], j)
|
||||
first, retry = step(job, 'Build arm64'), step(job, 'Retry arm64')
|
||||
self.assertIn("if: steps.arm64.outcome == 'failure'", step(job, 'Wait before'), j)
|
||||
self.assertIn("if: steps.arm64.outcome == 'failure'", retry, j)
|
||||
self.assertIn('id: arm64_retry', retry, j)
|
||||
self.assertIn('continue-on-error: true', retry, j)
|
||||
self.assertEqual(len(re.findall(r'Retry arm64', job)), 1, j) # exactly one retry
|
||||
for key in ('timeout-minutes', 'platforms', 'file', 'outputs'):
|
||||
a = re.search(r'(?m)^\s+%s: .*$' % key, first)
|
||||
b = re.search(r'(?m)^\s+%s: .*$' % key, retry)
|
||||
self.assertEqual(a and a.group(0), b and b.group(0), (j, key))
|
||||
ba = lambda x: re.search(r'build-args: \|\n((?: .*\n)+)', x).group(1)
|
||||
self.assertEqual(ba(first), ba(retry), j)
|
||||
|
||||
def test_matrix_families_cap_parallel_qemu_legs(self):
|
||||
for j, (_, _, matrix) in SLICE.items():
|
||||
if matrix:
|
||||
self.assertRegex(self.jobs[j], r'(?m)^ max-parallel: [12]$', j)
|
||||
|
||||
def test_assemble_guard_verify_and_override_wiring(self):
|
||||
want = {'Build-and-Push': ['/usr/sbin/httpd', '/usr/bin/php'],
|
||||
'Build-FPM-Images': ['/usr/sbin/php-fpm', '/usr/bin/php'],
|
||||
'Build-Shared-httpd': ['/usr/sbin/httpd']}
|
||||
for j, bins in want.items():
|
||||
st = step(self.jobs[j], 'Assemble')
|
||||
self.assertIn('--guard-existing-arm64', st, j)
|
||||
self.assertEqual(re.findall(r'--verify-bin (\S+)', st), bins, j)
|
||||
self.assertIn('ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }}', st, j)
|
||||
run = re.search(r'run: \|\n((?: .*\n?)+)', st).group(1)
|
||||
self.assertNotIn('${{', run, j)
|
||||
|
||||
def test_allow_amd64_only_input_is_a_string_default_false(self):
|
||||
head = TEXT.split('jobs:')[0]
|
||||
m = re.search(r'(?m)^ allow_amd64_only:\n((?: .*\n)+)', head)
|
||||
self.assertIsNotNone(m)
|
||||
self.assertIn("default: 'false'", m.group(1))
|
||||
self.assertIn('type: string', m.group(1))
|
||||
|
||||
def test_tag_scheme_unchanged(self):
|
||||
"""Run the assemble step's shell with a recording stub: exact tags per version."""
|
||||
for j, (repo, _, matrix) in SLICE.items():
|
||||
|
||||
Reference in new issue
Block a user