ci: arm64 policy - serialise legs, one retry, never drop existing arm64, ELF check #49

Merged
jknapp merged 2 commits from fix/arm64-policy into trunk 2026-11-20 05:13:13 +00:00
5 changed files with 44 additions and 20 deletions
Showing only changes of commit 344c5e8d0d - Show all commits

No files matched your search

+10 -8
View File
@@ -164,7 +164,7 @@ jobs:
needs: plan
if: needs.plan.outputs.cac != '[]'
runs-on: ubuntu-latest
timeout-minutes: 75
timeout-minutes: 130
strategy:
# One PHP version failing (e.g. an EOL stream) must not cancel the
# security builds of the others.
@@ -172,7 +172,8 @@ jobs:
# 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps
# (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded.
# Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether
# the runner honours it is unverified).
# the runner honours it is unverified: Gitea expands the matrix server-side). It is a
# per-job cap only: concurrency ACROSS jobs (cac + cac-fpm + shared-httpd) is not bounded.
max-parallel: 2
matrix:
phpver: ${{ fromJSON(needs.plan.outputs.cac) }}
@@ -339,7 +340,7 @@ jobs:
- name: Assemble index tag(s)
env:
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }}
ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }}
PHPVER: ${{ matrix.phpver }}
@@ -359,7 +360,7 @@ jobs:
needs: plan
if: needs.plan.outputs.cac_fpm != '[]'
runs-on: ubuntu-latest
timeout-minutes: 75
timeout-minutes: 130
strategy:
# One PHP version failing (e.g. an EOL stream) must not cancel the
# security builds of the others.
@@ -367,7 +368,8 @@ jobs:
# 2026-11-19: 4+4 QEMU arm64 legs ran at once and 6 of 8 failed on network steps
# (Remi / pkg-cache timeouts, truncated repomd.xml.asc); legs run alone succeeded.
# Caps concurrent legs of this family (max-parallel 1 = safest, slowest; whether
# the runner honours it is unverified).
# the runner honours it is unverified: Gitea expands the matrix server-side). It is a
# per-job cap only: concurrency ACROSS jobs (cac + cac-fpm + shared-httpd) is not bounded.
max-parallel: 2
matrix:
phpver: ${{ fromJSON(needs.plan.outputs.cac_fpm) }}
@@ -537,7 +539,7 @@ jobs:
- name: Assemble index tag(s)
env:
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }}
ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }}
PHPVER: ${{ matrix.phpver }}
@@ -871,7 +873,7 @@ jobs:
needs: plan
if: needs.plan.outputs.shared_httpd == 'true'
runs-on: ubuntu-latest
timeout-minutes: 50
timeout-minutes: 80
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -1000,7 +1002,7 @@ jobs:
- name: Assemble index tag(s)
env:
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}
ARM64_OUTCOME: ${{ (steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success') && 'success' || steps.arm64.outcome }}
ALLOW_AMD64_ONLY: ${{ github.event.inputs.allow_amd64_only }}
run: |
+1 -1
View File
@@ -5,4 +5,4 @@ The arm64 leg is best effort (QEMU on the x86 runner) but never silently lost:
- Matrix families run at most 2 legs at once (`max-parallel: 2`); 2026-11-19 showed that 8 concurrent QEMU legs fail on network steps.
- A failed arm64 leg is retried ONCE after 90 s (`Retry arm64`). Assemble uses the first success.
- Before publishing, `scripts/assemble-multiarch-index.sh --verify-bin` checks ELF machine types (amd64 = x86-64, arm64 = AArch64). A wrong arm64 counts as a failed leg; a wrong amd64 aborts.
- `--guard-existing-arm64`: if the build ends without arm64 and the current tag has an arm64 entry, the assemble step FAILS and publishes nothing (the amd64 digest is in the error). Options: re-dispatch `only=<leg>`, or dispatch with `allow_amd64_only=true` (security rebuilds) and then restore arm64 with the `arm64-attach` workflow. Push-triggered runs have no input, so the guard is always on there.
- `--guard-existing-arm64`: if the build ends without arm64 (failed, skipped or cancelled leg) and the current tag has an arm64 entry, the assemble step FAILS and publishes nothing (the amd64 digest is in the error). Options: re-dispatch `only=<leg>`, or dispatch with `allow_amd64_only=true` (security rebuilds) and then restore arm64 with the `arm64-attach` workflow. Push-triggered runs have no input, so the guard is always on there.
+4 -2
View File
@@ -68,7 +68,9 @@ class Wiring(unittest.TestCase):
at = int(re.search(r'timeout-minutes: (\d+)', step(job, 'Build arm64')).group(1))
self.assertGreaterEqual(at, 20, j) # T3: arm64 under qemu took 7m47s
self.assertGreater(jt, at + 20, j) # room for setup + amd64 + assemble
self.assertLessEqual(jt, 120, j)
self.assertLessEqual(jt, 150, j)
# the retry must fit: 2 arm64 attempts + 90 s pause + amd64 + verify/assemble margin
self.assertGreater(jt, 2 * at + 2 + {'Build-and-Push': 20, 'Build-FPM-Images': 20, 'Build-Shared-httpd': 10}[j] + 15, j)
def test_step_order_and_ids(self):
for j in SLICE:
@@ -124,7 +126,7 @@ class Wiring(unittest.TestCase):
for j, (repo, _, _) in SLICE.items():
st = step(self.jobs[j], 'Assemble')
self.assertIn('${{ steps.amd64.outputs.digest }}', st, j)
self.assertIn('${{ steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}', st, j)
self.assertIn("${{ steps.arm64.outcome == 'success' && steps.arm64.outputs.digest || steps.arm64_retry.outputs.digest }}", st, j)
self.assertIn("steps.arm64.outcome == 'success' || steps.arm64_retry.outcome == 'success'", st, j)
self.assertIn('scripts/assemble-multiarch-index.sh', st, j)
self.assertIn('--repo %s/%s' % (REG, repo), st, j)
+4 -2
View File
@@ -29,7 +29,7 @@ echo "$*" >>"$LOG"
case "$1" in
pull) exit "${PULL_RC:-0}" ;;
create) [ "${CREATE_RC:-0}" = 0 ] || { echo "no such image" >&2; exit 1; }; echo cid1 ;;
rm) ;;
rm|image) ;;
cp) python3 -I "$MKTAR" "${3#*:}" ;;
esac
STUB
@@ -55,8 +55,10 @@ printf '%s\n' /bin/a=notelf /bin/b=elf32 /bin/c=be /bin/d=dir /bin/e=missing >"$
for b in a b c d e; do chk "bad binary /bin/$b -> 1" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /bin/$b)" 1; done
chk "binary absent from the fixture -> 1" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/nothere)" 1
printf '%s\n' /usr/bin/php=b7 >"$T/fix"
chk "docker pull fails -> 1" "$(PULL_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 1
chk "docker pull fails (3 attempts) -> 4, not an arch verdict" "$(VERIFY_PULL_SLEEP=0 PULL_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 4
chk " pull was tried 3 times" "$(grep -c '^pull' "$LOG")" 3
chk "docker create fails -> 1" "$(CREATE_RC=1 run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php)" 1
chk "pulled image is removed afterwards" "$(run --image r/x@sha256:1 --platform linux/arm64 --bin /usr/bin/php >/dev/null; grep -c '^image rm r/x@sha256:1$' "$LOG")" 1
chk "--no-pull skips the pull" "$(run --image r/x@sha256:1 --platform linux/arm64 --no-pull --bin /usr/bin/php; grep -c '^pull' "$LOG" || true)" "0
0"
chk "usage: no --bin -> 2" "$(run --image r/x --platform linux/arm64)" 2
+25 -7
View File
@@ -12,7 +12,7 @@ no QEMU execution is needed.
Exit 0 = every --bin is an ELF64 little-endian binary of the platform's machine type.
Exit 1 = a binary is missing, not an ELF, or of the wrong architecture (or docker failed).
Exit 2 = usage.
Exit 2 = usage. Exit 4 = could not pull the image (3 attempts): no architecture verdict.
Env: DOCKER overrides the docker command (tests use a stub). Nothing in REF is ever run.
"""
import argparse
@@ -21,6 +21,8 @@ import shlex
import subprocess
import sys
import tarfile
import threading
import time
MACHINES = {'linux/arm64': (0xB7, 'AArch64'), 'linux/amd64': (0x3E, 'x86-64')}
NAMES = {0x3E: 'x86-64', 0xB7: 'AArch64', 0x03: 'x86', 0x28: 'ARM', 0xF3: 'RISC-V', 0x15: 'PowerPC64', 0x16: 'S390'}
@@ -42,14 +44,19 @@ def docker_cmd():
return shlex.split(os.environ.get('DOCKER') or 'docker')
def run(args, **kw):
return subprocess.run(docker_cmd() + args, capture_output=True, **kw)
def run(args, timeout=300):
try:
return subprocess.run(docker_cmd() + args, capture_output=True, timeout=timeout)
except subprocess.TimeoutExpired:
return subprocess.CompletedProcess(args, 124, b'', b'timed out after %ds' % timeout)
def first_bytes_of(cid, path):
"""First 64 bytes of the regular file at path (symlinks followed) inside container cid."""
p = subprocess.Popen(docker_cmd() + ['cp', '-L', '%s:%s' % (cid, path), '-'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
timer = threading.Timer(120, p.kill) # a stalled docker cp must not hang the job
timer.start()
try:
tf = tarfile.open(fileobj=p.stdout, mode='r|')
m = tf.next()
@@ -60,6 +67,7 @@ def first_bytes_of(cid, path):
err = p.stderr.read(300).decode('utf-8', 'replace').strip()
raise ValueError('cannot read %s: %s %s' % (path, e, err))
finally:
timer.cancel()
p.kill()
p.wait()
@@ -73,11 +81,19 @@ def main(argv):
a = ap.parse_args(argv)
want, label = MACHINES[a.platform]
pulled = False
if not a.no_pull:
r = run(['pull', '--quiet', '--platform', a.platform, a.image])
r = None
for attempt in range(3):
r = run(['pull', '--quiet', '--platform', a.platform, a.image], timeout=900)
if r.returncode == 0:
break
time.sleep(float(os.environ.get('VERIFY_PULL_SLEEP', '15')))
if r.returncode != 0:
print('verify-image-arch: docker pull failed: %s' % r.stderr.decode('utf-8', 'replace')[:300].strip(), file=sys.stderr)
return 1
print('verify-image-arch: could not pull %s after 3 attempts (not an architecture finding): %s' % (
a.image, r.stderr.decode('utf-8', 'replace')[:300].strip()), file=sys.stderr)
return 4
pulled = True
r = run(['create', '--platform', a.platform, '--entrypoint', '/nonexistent', a.image])
cid = r.stdout.decode().strip()
if r.returncode != 0 or not cid:
@@ -99,7 +115,9 @@ def main(argv):
else:
print('OK %s %s %s: e_machine=0x%X (%s)' % (a.image, a.platform, path, m, label))
finally:
run(['rm', '-f', cid])
run(['rm', '-f', cid], timeout=120)
if pulled:
run(['image', 'rm', a.image], timeout=120) # keep the runner disk from filling
return rc