From 8644d8e38c9ab2cec5f6e20000fdead4ebd88351 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 21:10:08 -0800 Subject: [PATCH] ci: address Opus review of the arm64 policy PR - job timeouts 130 (cac, cac-fpm) / 80 (shared-httpd) so the retry fits; test pins the budget - verify-image-arch.py: bounded docker calls, pull retried 3x, exit 4 'could not pull' (not an arch verdict), pulled image removed afterwards - ARM64_DIGEST only taken from the first attempt when it succeeded - comments/docs: max-parallel is per-job and unverified; guard also fires on skipped/cancelled Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- .gitea/workflows/build-push.yaml | 18 +++++++------ docs/arm64-publish-policy.md | 2 +- scripts/tests/multiarch-workflow.test.py | 6 +++-- scripts/tests/verify-image-arch.test.sh | 6 +++-- scripts/verify-image-arch.py | 32 ++++++++++++++++++------ 5 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index dafcac0..ef38bb0 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -164,7 +164,7 @@ jobs: needs: plan if: needs.plan.outputs.cac != '[]' runs-on: ubuntu-latest - timeout-minutes: 75 + timeout-minutes: 130 strategy: # One PHP version failing (e.g. an EOL stream) must not cancel the # security builds of the others. @@ -172,7 +172,8 @@ jobs: # 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps # (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded. # Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether - # the runner honours it is unverified). + # the runner honours it is unverified: Gitea expands the matrix server-side). It is a + # per-job cap only: concurrency ACROSS jobs (cac + cac-fpm + shared-httpd) is not bounded. max-parallel: 2 matrix: phpver: ${{ fromJSON(needs.plan.outputs.cac) }} @@ -339,7 +340,7 @@ jobs: - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} PHPVER: ${{ matrix.phpver }} @@ -359,7 +360,7 @@ jobs: needs: plan if: needs.plan.outputs.cac_fpm != '[]' runs-on: ubuntu-latest - timeout-minutes: 75 + timeout-minutes: 130 strategy: # One PHP version failing (e.g. an EOL stream) must not cancel the # security builds of the others. @@ -367,7 +368,8 @@ jobs: # 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps # (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded. # Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether - # the runner honours it is unverified). + # the runner honours it is unverified: Gitea expands the matrix server-side). It is a + # per-job cap only: concurrency ACROSS jobs (cac + cac-fpm + shared-httpd) is not bounded. max-parallel: 2 matrix: phpver: ${{ fromJSON(needs.plan.outputs.cac_fpm) }} @@ -537,7 +539,7 @@ jobs: - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} PHPVER: ${{ matrix.phpver }} @@ -871,7 +873,7 @@ jobs: needs: plan if: needs.plan.outputs.shared_httpd == 'true' runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 80 steps: - name: Checkout uses: actions/checkout@v4 @@ -1000,7 +1002,7 @@ jobs: - name: Assemble index tag(s) env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} - ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} + ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }} ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }} ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }} run: | diff --git a/docs/arm64-publish-policy.md b/docs/arm64-publish-policy.md index b239843..0a34969 100644 --- a/docs/arm64-publish-policy.md +++ b/docs/arm64-publish-policy.md @@ -5,4 +5,4 @@ The arm64 leg is best effort (QEMU on the x86 runner) but never silently lost: - Matrix families run at most 2 legs at once (`max-parallel: 2`); 2026-11-19 showed that 8 concurrent QEMU legs fail on network steps. - A failed arm64 leg is retried ONCE after 90 s (`Retry arm64`). Assemble uses the first success. - Before publishing, `scripts/assemble-multiarch-index.sh --verify-bin` checks ELF machine types (amd64 = x86-64, arm64 = AArch64). A wrong arm64 counts as a failed leg; a wrong amd64 aborts. -- `--guard-existing-arm64`: if the build ends without arm64 and the current tag has an arm64 entry, the assemble step FAILS and publishes nothing (the amd64 digest is in the error). Options: re-dispatch `only=`, or dispatch with `allow_amd64_only=true` (security rebuilds) and then restore arm64 with the `arm64-attach` workflow. Push-triggered runs have no input, so the guard is always on there. +- `--guard-existing-arm64`: if the build ends without arm64 (failed, skipped or cancelled leg) and the current tag has an arm64 entry, the assemble step FAILS and publishes nothing (the amd64 digest is in the error). Options: re-dispatch `only=`, or dispatch with `allow_amd64_only=true` (security rebuilds) and then restore arm64 with the `arm64-attach` workflow. Push-triggered runs have no input, so the guard is always on there. diff --git a/scripts/tests/multiarch-workflow.test.py b/scripts/tests/multiarch-workflow.test.py index acaf02f..14b019d 100755 --- a/scripts/tests/multiarch-workflow.test.py +++ b/scripts/tests/multiarch-workflow.test.py @@ -68,7 +68,9 @@ class Wiring(unittest.TestCase): at = int(re.search(r'timeout-minutes: (\d+)', step(job, 'Build arm64')).group(1)) self.assertGreaterEqual(at, 20, j) # T3: arm64 under qemu took 7m47s self.assertGreater(jt, at + 20, j) # room for setup + amd64 + assemble - self.assertLessEqual(jt, 120, j) + self.assertLessEqual(jt, 150, j) + # the retry must fit: 2 arm64 attempts + 90 s pause + amd64 + verify/assemble margin + self.assertGreater(jt, 2 * at + 2 + {'Build-and-Push': 20, 'Build-FPM-Images': 20, 'Build-Shared-httpd': 10}[j] + 15, j) def test_step_order_and_ids(self): for j in SLICE: @@ -124,7 +126,7 @@ class Wiring(unittest.TestCase): for j, (repo, _, _) in SLICE.items(): st = step(self.jobs[j], 'Assemble') self.assertIn('${{ steps.amd64.outputs.digest }}', st, j) - self.assertIn('${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}', st, j) + self.assertIn("${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}", st, j) self.assertIn("steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success'", st, j) self.assertIn('scripts/assemble-multiarch-index.sh', st, j) self.assertIn('--repo %s/%s' % (REG, repo), st, j) diff --git a/scripts/tests/verify-image-arch.test.sh b/scripts/tests/verify-image-arch.test.sh index a3b3d59..1954bf6 100755 --- a/scripts/tests/verify-image-arch.test.sh +++ b/scripts/tests/verify-image-arch.test.sh @@ -29,7 +29,7 @@ echo "$*" >>"$LOG" case "$1" in pull) exit "${PULL_RC:-0}" ;; create) [ "${CREATE_RC:-0}" = 0 ] || { echo "no such image" >&2; exit 1; }; echo cid1 ;; - rm) ;; + rm|image) ;; cp) python3 -I "$MKTAR" "${3#*:}" ;; esac STUB @@ -55,8 +55,10 @@ printf '%s\n' /bin/a=notelf /bin/b=elf32 /bin/c=be /bin/d=dir /bin/e=missing >"$ for b in a b c d e; do chk "bad binary /bin/$b -> 1" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /bin/$b)" 1; done chk "binary absent from the fixture -> 1" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/nothere)" 1 printf '%s\n' /usr/bin/php=b7 >"$T/fix" -chk "docker pull fails -> 1" "$(PULL_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 1 +chk "docker pull fails (3 attempts) -> 4, not an arch verdict" "$(VERIFY_PULL_SLEEP=0 PULL_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 4 +chk " pull was tried 3 times" "$(grep -c '^pull' "$LOG")" 3 chk "docker create fails -> 1" "$(CREATE_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 1 +chk "pulled image is removed afterwards" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php >/dev/null; grep -c '^image rm r/x@sha256:1$' "$LOG")" 1 chk "--no-pull skips the pull" "$(run --image r/x@sha256:1 --platform linux/arm64 --no-pull --bin /usr/bin/php; grep -c '^pull' "$LOG" || true)" "0 0" chk "usage: no --bin -> 2" "$(run --image r/x --platform linux/arm64)" 2 diff --git a/scripts/verify-image-arch.py b/scripts/verify-image-arch.py index d8eb4ad..bcc3352 100755 --- a/scripts/verify-image-arch.py +++ b/scripts/verify-image-arch.py @@ -12,7 +12,7 @@ no QEMU execution is needed. Exit 0 = every --bin is an ELF64 little-endian binary of the platform's machine type. Exit 1 = a binary is missing, not an ELF, or of the wrong architecture (or docker failed). -Exit 2 = usage. +Exit 2 = usage. Exit 4 = could not pull the image (3 attempts): no architecture verdict. Env: DOCKER overrides the docker command (tests use a stub). Nothing in REF is ever run. """ import argparse @@ -21,6 +21,8 @@ import shlex import subprocess import sys import tarfile +import threading +import time MACHINES = {'linux/arm64': (0xB7, 'AArch64'), 'linux/amd64': (0x3E, 'x86-64')} NAMES = {0x3E: 'x86-64', 0xB7: 'AArch64', 0x03: 'x86', 0x28: 'ARM', 0xF3: 'RISC-V', 0x15: 'PowerPC64', 0x16: 'S390'} @@ -42,14 +44,19 @@ def docker_cmd(): return shlex.split(os.environ.get('DOCKER') or 'docker') -def run(args, **kw): - return subprocess.run(docker_cmd() + args, capture_output=True, **kw) +def run(args, timeout=300): + try: + return subprocess.run(docker_cmd() + args, capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(args, 124, b'', b'timed out after %ds' % timeout) def first_bytes_of(cid, path): """First 64 bytes of the regular file at path (symlinks followed) inside container cid.""" p = subprocess.Popen(docker_cmd() + ['cp', '-L', '%s:%s' % (cid, path), '-'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + timer = threading.Timer(120, p.kill) # a stalled docker cp must not hang the job + timer.start() try: tf = tarfile.open(fileobj=p.stdout, mode='r|') m = tf.next() @@ -60,6 +67,7 @@ def first_bytes_of(cid, path): err = p.stderr.read(300).decode('utf-8', 'replace').strip() raise ValueError('cannot read %s: %s %s' % (path, e, err)) finally: + timer.cancel() p.kill() p.wait() @@ -73,11 +81,19 @@ def main(argv): a = ap.parse_args(argv) want, label = MACHINES[a.platform] + pulled = False if not a.no_pull: - r = run(['pull', '--quiet', '--platform', a.platform, a.image]) + r = None + for attempt in range(3): + r = run(['pull', '--quiet', '--platform', a.platform, a.image], timeout=900) + if r.returncode == 0: + break + time.sleep(float(os.environ.get('VERIFY_PULL_SLEEP', '15'))) if r.returncode != 0: - print('verify-image-arch: docker pull failed: %s' % r.stderr.decode('utf-8', 'replace')[:300].strip(), file=sys.stderr) - return 1 + print('verify-image-arch: could not pull %s after 3 attempts (not an architecture finding): %s' % ( + a.image, r.stderr.decode('utf-8', 'replace')[:300].strip()), file=sys.stderr) + return 4 + pulled = True r = run(['create', '--platform', a.platform, '--entrypoint', '/nonexistent', a.image]) cid = r.stdout.decode().strip() if r.returncode != 0 or not cid: @@ -99,7 +115,9 @@ def main(argv): else: print('OK %s %s %s: e_machine=0x%X (%s)' % (a.image, a.platform, path, m, label)) finally: - run(['rm', '-f', cid]) + run(['rm', '-f', cid], timeout=120) + if pulled: + run(['image', 'rm', a.image], timeout=120) # keep the runner disk from filling return rc