ci: make tag verification independent of buildx version

buildx v0.28.0 ignores `imagetools inspect --format '{{.Manifest.Digest}}'`
and prints the whole inspect text, so the verify step rejected every ref as a
"bad digest" (CI run 5381). Move the gate into scripts/verify-published-tags.sh:
the index digest is the sha256 of `inspect --raw`, cross-checked against the
anchored `Digest:` line and a bare --format output when present; platforms are
read from the raw JSON. All other checks (every tag in every registry, platform
match, one digest across registries, any miss fails) are unchanged. Adds an
offline test with a stub emulating 0.28 and 0.38 output.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
shadowdaoandClaude Sonnet 5.5 committed 2026-11-16 22:30:21 -08:00
1 parent ba8722b0b7
commit aebb68fe9d
4 files changed
+301 -23

No files matched your search

+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""Offline tests for scripts/verify-published-tags.sh (final publish gate).
CI run 5381 failed because buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'`
and prints the whole inspect text, while buildx 0.38 prints a bare digest. A stub
`imagetools` emulates BOTH output styles (plus `--raw`), so no registry or Docker
is needed. Held invariants:
- the 0.28-style (full text) and 0.38-style (bare digest) outputs both verify
- a reported digest that disagrees with the sha256 of the raw manifest fails
- a missing tag, a platform mismatch, or different digests across registries /
tags fail the job
- build-push.yaml's verify step uses the script, and release/* + rc-* tags
resolve to ONLY <version>-rc and <sha> (never latest)
Running: python3 scripts/test-verify-published-tags.py (needs jq)
"""
import hashlib
import json
import os
import re
import stat
import subprocess
import tempfile
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
SCRIPT = os.path.join(HERE, 'verify-published-tags.sh')
WORKFLOW = os.path.join(ROOT, '.gitea', 'workflows', 'build-push.yaml')
STUB = r'''#!/usr/bin/env python3
import hashlib, json, os, sys
a = sys.argv[1:]
mode = os.environ['STUB_MODE'] # v028 | v038
table = json.load(open(os.environ['STUB_TABLE'])) # ref -> {"raw": str, "lie": optional digest}
assert a[0] == 'inspect'
ref = [x for x in a[1:] if not x.startswith('-') and not x.startswith('{{')][0]
ent = table.get(ref)
if ent is None:
sys.stderr.write('ERROR: %s: not found\n' % ref); sys.exit(1)
raw = ent['raw'].encode() + (b'\n' if ent.get('nl') else b'')
digest = ent.get('lie') or 'sha256:' + hashlib.sha256(ent['raw'].encode()).hexdigest()
if '--raw' in a:
sys.stdout.buffer.write(raw); sys.exit(0)
if '--format' in a:
if mode == 'v038':
print(digest)
else: # 0.28: --format ignored, whole text printed
print('Name: ' + ref); print('MediaType: application/vnd.oci.image.index.v1+json')
print('Digest: ' + digest)
sys.exit(0)
print('Name: ' + ref)
print('MediaType: application/vnd.oci.image.index.v1+json')
print('Digest: ' + digest)
print(' ')
print('Manifests: ')
for m in json.loads(ent['raw'])['manifests']:
print(' Name: %s@%s' % (ref, m['digest']))
print(' Digest: sha256:' + 'f' * 64) # indented: must never be mistaken for the index digest
print(' Platform: %s/%s' % (m['platform']['os'], m['platform']['architecture']))
'''
def index(archs, salt=''):
ms = [{'mediaType': 'application/vnd.oci.image.manifest.v1+json',
'digest': 'sha256:' + hashlib.sha256((x + salt).encode()).hexdigest(), 'size': 1,
'platform': {'architecture': x, 'os': 'linux'}} for x in archs]
ms.append({'mediaType': 'application/vnd.oci.image.manifest.v1+json',
'digest': 'sha256:' + '7' * 64, 'size': 1,
'platform': {'architecture': 'unknown', 'os': 'unknown'},
'annotations': {'vnd.docker.reference.type': 'attestation-manifest'}})
return json.dumps({'schemaVersion': 2, 'mediaType': 'application/vnd.oci.image.index.v1+json',
'manifests': ms}, separators=(',', ':'))
REPOS = ['reg.one/a/img', 'reg.two/b/img']
TAGS = ['1.2-rc', 'abc123']
class VerifyTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.stub = os.path.join(self.tmp.name, 'stub')
open(self.stub, 'w').write(STUB)
os.chmod(self.stub, os.stat(self.stub).st_mode | stat.S_IEXEC)
self.table = os.path.join(self.tmp.name, 'table.json')
def tearDown(self):
self.tmp.cleanup()
def table_for(self, raw, **over):
t = {'%s:%s' % (r, g): {'raw': raw} for r in REPOS for g in TAGS}
for k, v in over.items():
ref = k.replace('__', ':').replace('_', '/', 0)
t[ref] = v
return t
def run_script(self, table, mode, platforms='linux/amd64,linux/arm64', tags=TAGS):
json.dump(table, open(self.table, 'w'))
args = [SCRIPT, '--platforms', platforms]
for r in REPOS:
args += ['--repo', r]
for t in tags:
args += ['--tag', t]
env = dict(os.environ, IMAGETOOLS=self.stub, STUB_MODE=mode, STUB_TABLE=self.table)
return subprocess.run(args, env=env, capture_output=True, text=True, timeout=60)
def test_buildx_028_full_text_output_verifies(self):
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v028')
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
self.assertIn('verified:', r.stdout)
self.assertNotIn('bad digest', r.stdout)
def test_buildx_038_bare_digest_output_verifies(self):
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v038')
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
def test_trailing_newline_on_raw_output_tolerated(self):
raw = index(['amd64', 'arm64'])
t = {'%s:%s' % (r, g): {'raw': raw, 'nl': True} for r in REPOS for g in TAGS}
self.assertEqual(self.run_script(t, 'v028').returncode, 0)
def test_amd64_only_publish_matches_platforms(self):
raw = index(['amd64'])
self.assertEqual(self.run_script(self.table_for(raw), 'v028', 'linux/amd64').returncode, 0)
r = self.run_script(self.table_for(raw), 'v028', 'linux/amd64,linux/arm64')
self.assertEqual(r.returncode, 1)
self.assertIn('platform mismatch', r.stdout)
def test_missing_tag_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
del t['reg.two/b/img:abc123']
r = self.run_script(t, 'v038')
self.assertEqual(r.returncode, 1)
self.assertIn('MISSING', r.stdout)
def test_different_digest_across_registries_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
t['reg.two/b/img:1.2-rc'] = {'raw': index(['amd64', 'arm64'], salt='other')}
for mode in ('v028', 'v038'):
r = self.run_script(t, mode)
self.assertEqual(r.returncode, 1, mode)
self.assertIn('digest mismatch', r.stdout)
def test_reported_digest_disagreeing_with_raw_manifest_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
t['reg.one/a/img:1.2-rc']['lie'] = 'sha256:' + '0' * 64
for mode in ('v028', 'v038'):
r = self.run_script(t, mode)
self.assertEqual(r.returncode, 1, mode)
self.assertIn('cross-check FAILED', r.stdout)
def test_garbage_manifest_fails(self):
t = self.table_for('not json')
self.assertEqual(self.run_script(t, 'v038').returncode, 1)
class WorkflowTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
import yaml
cls.src = open(WORKFLOW).read()
cls.doc = yaml.safe_load(cls.src)
cls.steps = cls.doc['jobs']['Build-and-Push']['steps']
def step(self, name):
return [s for s in self.steps if s.get('name') == name][0]
def test_verify_step_uses_script_not_format_digest(self):
run = self.step('Verify published tags in every registry')['run']
self.assertIn('scripts/verify-published-tags.sh', run)
self.assertNotIn('{{.Manifest.Digest}}', run)
def test_run_blocks_are_valid_bash(self):
for s in self.steps:
if 'run' in s:
p = subprocess.run(['bash', '-n'], input=s['run'], text=True, capture_output=True)
self.assertEqual(p.returncode, 0, '%s: %s' % (s.get('name'), p.stderr))
def test_release_and_rc_never_publish_latest(self):
run = self.step('Resolve tags')['run']
m = re.search(r'refs/heads/release/\*\|refs/tags/rc-\*\)\s*\n\s*tags="([^"]*)"', run)
self.assertIsNotNone(m)
self.assertEqual(m.group(1), '${VERSION}-rc ${SHA}')
main = re.search(r'refs/heads/main\)\s*\n\s*tags="([^"]*)"', run)
self.assertIsNotNone(main)
self.assertEqual(main.group(1), 'latest ${VERSION} ${SHA}')
# latest appears in no other tag assignment
self.assertEqual(len(re.findall(r'tags="[^"]*latest', run)), 1)
if __name__ == '__main__':
unittest.main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# Final publish gate: every <repo>:<tag> must resolve, carry exactly the
# platforms the assemble step reported, and have ONE index digest across all
# tags and all registries. Any miss or mismatch exits 1.
#
# verify-published-tags.sh --platforms linux/amd64,linux/arm64 \
# --repo <registry/path> [--repo ...] --tag <tag> [--tag ...]
#
# The index digest is computed in a buildx-VERSION-INDEPENDENT way. CI run 5381
# showed buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'` and prints the
# whole inspect text, while 0.38 prints a bare digest. So the digest comes from:
# 1. sha256 of the raw manifest bytes (`inspect --raw`) - always, authoritative
# (a content digest IS the sha256 of the manifest bytes);
# 2. the strictly anchored `Digest:` line of the default text output, if present;
# 3. the `--format '{{.Manifest.Digest}}'` output, ONLY if it is a bare digest
# (0.28's full-text output is ignored here, not an error).
# Every source that is available must agree, else the ref fails.
# Platforms are read from the raw JSON (also version independent).
#
# Env: IMAGETOOLS overrides the command (default "docker buildx imagetools");
# the CI step sets it to "$BUILDX imagetools"; the test suite uses a stub.
set -uo pipefail
IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools}
platforms=""; repos=(); tags=()
while [ $# -gt 0 ]; do
case "$1" in
--platforms) platforms=${2-}; shift 2 ;;
--repo) repos+=("${2-}"); shift 2 ;;
--tag) tags+=("${2-}"); shift 2 ;;
*) echo "verify-published-tags: unknown argument: $1" >&2; exit 2 ;;
esac
done
[ -n "$platforms" ] && [ ${#repos[@]} -gt 0 ] && [ ${#tags[@]} -gt 0 ] \
|| { echo "::error::verify-published-tags: --platforms, --repo and --tag are required"; exit 2; }
command -v jq >/dev/null || { echo "::error::verify-published-tags: jq is required"; exit 2; }
want=$(printf '%s' "$platforms" | tr ',' '\n' | sort | tr '\n' ' ')
work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
digest_re='^sha256:[0-9a-f]{64}$'
ref_digest=""; ref_name=""; fail=0
for r in "${repos[@]}"; do
for t in "${tags[@]}"; do
ref="${r}:${t}"
raw="$work/raw"
if ! timeout 90 $IMAGETOOLS inspect --raw "$ref" >"$raw" 2>"$work/err" || [ ! -s "$raw" ]; then
echo "::error::MISSING or unreadable: ${ref}: $(head -c 300 "$work/err")"; fail=1; continue
fi
# Raw bytes as served, and with trailing newlines stripped (some buildx
# versions append one when printing); the content digest is one of the two.
h1="sha256:$(sha256sum <"$raw" | cut -d' ' -f1)"
h2="sha256:$(printf '%s' "$(cat "$raw")" | sha256sum | cut -d' ' -f1)"
# Source 2: strictly anchored top-level "Digest:" line of the text output.
textd=""
if timeout 90 $IMAGETOOLS inspect "$ref" >"$work/text" 2>/dev/null; then
mapfile -t lines < <(grep -E '^Digest:[[:space:]]+sha256:[0-9a-f]{64}[[:space:]]*$' "$work/text" | sed -E 's/^Digest:[[:space:]]+//; s/[[:space:]]+$//')
[ ${#lines[@]} -eq 1 ] && textd=${lines[0]}
fi
# Source 3: bare --format output, only when it really is a bare digest.
fmtd=""
if out=$(timeout 90 $IMAGETOOLS inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null); then
out=$(printf '%s' "$out" | tr -d '[:space:]')
[[ "$out" =~ $digest_re ]] && fmtd=$out
fi
d=""
for cand in "$textd" "$fmtd"; do
[ -n "$cand" ] || continue
if [ "$cand" != "$h1" ] && [ "$cand" != "$h2" ]; then
echo "::error::digest cross-check FAILED for ${ref}: reported ${cand}, raw manifest hashes to ${h1}"; fail=1; d="!"; break
fi
d=$cand
done
[ "$d" = "!" ] && continue
[ -n "$d" ] || d=$h2
[[ "$d" =~ $digest_re ]] || { echo "::error::bad digest for ${ref}: ${d}"; fail=1; continue; }
if [ -z "$textd$fmtd" ]; then
echo "::warning::no reported digest for ${ref}; using sha256 of the raw manifest"
fi
if ! got=$(jq -r '.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"' "$raw" 2>/dev/null | sort | tr '\n' ' '); then
echo "::error::unparseable manifest for ${ref}"; fail=1; continue
fi
if [ "$got" != "$want" ]; then
echo "::error::platform mismatch for ${ref}: have [${got}] want [${want}]"; fail=1
fi
echo "ok ${ref} ${d} [${got}]"
if [ -z "$ref_digest" ]; then ref_digest="$d"; ref_name="$ref"
elif [ "$d" != "$ref_digest" ]; then
echo "::error::digest mismatch: ${ref}=${d} vs ${ref_name}=${ref_digest}"; fail=1
fi
done
done
[ "$fail" = 0 ] || { echo "::error::tag verification FAILED"; exit 1; }
echo "verified: ${tags[*]} in ${#repos[@]} registries, digest ${ref_digest}, platforms ${platforms}"