diff --git a/.gitea/scripts/mirror-index-copy.sh b/.gitea/scripts/mirror-index-copy.sh new file mode 100755 index 0000000..095bbcb --- /dev/null +++ b/.gitea/scripts/mirror-index-copy.sh @@ -0,0 +1,64 @@ +#!/bin/bash +## mirror-index-copy.sh SRC DST — CI only (not copied into images). +## SRC registry/path@sha256: (digest-pinned upstream reference) +## DST registry/path: (the mirror tag consumers FROM) +## +## Registry-to-registry copy of the WHOLE upstream image index (every +## platform), then assertions. No local `docker pull`, so nothing is flattened +## to the runner's platform (the old pull/tag/push step re-published only the +## amd64 manifest, leaving the mirror without an arm64 variant). +## +## Ordering (protects a previously multi-arch DST tag): +## 1. copy SRC -> DST_REPO:-staging (real tag untouched) +## 2. assert staging: index lists every REQUIRED platform and its digest +## equals the digest in SRC +## 3. only then point DST at that digest (registry-side, no data transfer) +## 4. re-assert on DST +## A failure at 1/2 exits non-zero before the real tag is touched. A failure +## at 4 cannot be rolled back by this script (the tag already moved) but it is +## loud and the same digest was already proven at step 2. +## +## Env: REQUIRED_PLATFORMS (default "linux/amd64 linux/arm64"). +## IMAGETOOLS (default "docker buildx imagetools"), for tests/skopeo-free CI. +## (buildx talks plain HTTP to a localhost registry, so the test needs no flags.) +set -euo pipefail + +SRC="${1:?usage: mirror-index-copy.sh SRC@sha256:DIGEST DST_REPO:TAG}" +DST="${2:?usage: mirror-index-copy.sh SRC@sha256:DIGEST DST_REPO:TAG}" +REQUIRED_PLATFORMS="${REQUIRED_PLATFORMS:-linux/amd64 linux/arm64}" +read -r -a IMGTOOLS <<<"${IMAGETOOLS:-docker buildx imagetools}" + +fail() { echo "::error::mirror-index-copy: $*" >&2; exit 1; } + +# Validate shapes (registry host may carry :port, e.g. localhost:5055). +[[ "$SRC" =~ ^[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || fail "bad source reference '${SRC}'" +[[ "$DST" =~ ^[a-z0-9][a-z0-9._:/-]*:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || fail "bad destination '${DST}'" +src_digest="${SRC#*@}" +dst_repo="${DST%:*}"; dst_tag="${DST##*:}" +STAGING="${dst_repo}:${dst_tag}-staging" + +# assert_ref REF: REF must be an index listing every required platform, and +# its manifest digest must equal the upstream index digest. +assert_ref() { + local ref="$1" raw have want got + raw=$("${IMGTOOLS[@]}" inspect "$ref" --raw) || fail "cannot inspect ${ref}" + have=$(jq -r '[.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"] | unique | join(" ")' <<<"$raw") + [ -n "$have" ] || fail "${ref} is not a multi-platform index (single-platform manifest?); platforms: none" + for want in $REQUIRED_PLATFORMS; do + case " $have " in *" $want "*) ;; *) fail "${ref} is missing ${want} (has: ${have})" ;; esac + done + got=$("${IMGTOOLS[@]}" inspect "$ref" --format '{{.Manifest.Digest}}') || fail "cannot read digest of ${ref}" + [ "$got" = "$src_digest" ] || fail "${ref} digest ${got} != upstream ${src_digest}" + echo "ok: ${ref} platforms=[${have}] digest=${got}" +} + +echo "::group::copy index ${SRC} -> ${STAGING}" +"${IMGTOOLS[@]}" create -t "$STAGING" "$SRC" +echo "::endgroup::" +assert_ref "$STAGING" + +echo "::group::promote staging -> ${DST}" +"${IMGTOOLS[@]}" create -t "$DST" "${dst_repo}@${src_digest}" +echo "::endgroup::" +assert_ref "$DST" +echo "mirror ok: ${DST} == ${src_digest}" diff --git a/.gitea/scripts/test-mirror-index-copy.sh b/.gitea/scripts/test-mirror-index-copy.sh new file mode 100755 index 0000000..8096310 --- /dev/null +++ b/.gitea/scripts/test-mirror-index-copy.sh @@ -0,0 +1,56 @@ +#!/bin/bash +## Test for mirror-index-copy.sh. CI/dev only; needs docker + buildx + jq and a +## THROWAWAY registry on REG (default localhost:5055): +## docker run -d --name mirror-test-reg -p 127.0.0.1:5055:5000 registry:2 +## (remove it afterwards). Never point REG at a real registry. Builds only +## FROM scratch images, so it needs no network beyond the registry. +## Set MIRROR_TEST_UPSTREAM=docker.io/library/python@sha256:... to also run the +## copy against a real multi-arch upstream digest (anonymous read-only pull). +set -uo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +REG="${REG:-localhost:5055}" +case "$REG" in localhost:*|127.0.0.1:*) ;; *) echo "REG must be a localhost throwaway registry"; exit 2;; esac +curl -fsS -m5 "http://${REG}/v2/" >/dev/null || { echo "no registry on ${REG}"; exit 2; } +run="t$$"; work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +pass=0; failn=0 +ok() { echo "PASS: $1"; pass=$((pass+1)); } +bad() { echo "FAIL: $1"; failn=$((failn+1)); } +digest() { docker buildx imagetools inspect "$1" --format '{{.Manifest.Digest}}'; } + +mkimg() { # mkimg REPO:TAG PLATFORM -> tiny FROM-scratch image + printf 'FROM scratch\nLABEL t=%s\n' "$2-$run" >"$work/Dockerfile" + docker buildx build -q --platform "$2" --provenance=false --sbom=false \ + --output "type=image,name=${1},push=true" "$work" >/dev/null || { echo "build $1 failed"; exit 2; } +} +mkimg "${REG}/${run}/amd64:v" linux/amd64 +mkimg "${REG}/${run}/arm64:v" linux/arm64 +A=$(digest "${REG}/${run}/amd64:v"); B=$(digest "${REG}/${run}/arm64:v") +# synthetic two-platform upstream index, and a single-platform one +docker buildx imagetools create -t "${REG}/${run}/multi:v" "${REG}/${run}/amd64@${A}" "${REG}/${run}/arm64@${B}" >/dev/null 2>&1 || exit 2 +MULTI=$(digest "${REG}/${run}/multi:v"); SINGLE="$A" + +# 1. two-platform source: pass, digest preserved, tag lands on source digest +if "$here/mirror-index-copy.sh" "${REG}/${run}/multi@${MULTI}" "${REG}/${run}/dst:1" >"$work/o1" 2>&1 \ + && [ "$(digest "${REG}/${run}/dst:1")" = "$MULTI" ]; then ok "multi-arch copy passes and digest round-trips ($MULTI)"; else bad "multi-arch copy"; cat "$work/o1"; fi + +# 2. single-platform source: must FAIL loudly and must not create the real tag +if "$here/mirror-index-copy.sh" "${REG}/${run}/amd64@${SINGLE}" "${REG}/${run}/dst2:1" >"$work/o2" 2>&1; then bad "single-platform source was accepted"; else + grep -q 'is not a multi-platform index\|missing linux/arm64' "$work/o2" && ok "single-platform source rejected: $(grep -m1 '::error::' "$work/o2")" || { bad "failed for the wrong reason"; cat "$work/o2"; } + if docker buildx imagetools inspect "${REG}/${run}/dst2:1" >/dev/null 2>&1; then bad "real tag created despite failure"; else ok "real tag not created on failure"; fi +fi + +# 3. existing multi-arch tag must survive a failing (single-platform) refresh +before=$(digest "${REG}/${run}/dst:1") +if "$here/mirror-index-copy.sh" "${REG}/${run}/amd64@${SINGLE}" "${REG}/${run}/dst:1" >"$work/o3" 2>&1; then bad "downgrade accepted"; else + [ "$(digest "${REG}/${run}/dst:1")" = "$before" ] && ok "previously multi-arch :tag untouched after failed refresh" || bad ":tag moved to single-platform"; fi + +# 4. malformed references are refused before any registry traffic +"$here/mirror-index-copy.sh" "${REG}/x:tag" "${REG}/${run}/d:1" >/dev/null 2>&1 && bad "tag-only source accepted" || ok "non-digest source refused" +"$here/mirror-index-copy.sh" "${REG}/x@sha256:$(printf 'a%.0s' {1..64})" 'bad dst' >/dev/null 2>&1 && bad "bad dst accepted" || ok "bad destination refused" + +# 5. optional: real upstream digest +if [ -n "${MIRROR_TEST_UPSTREAM:-}" ]; then + u="${MIRROR_TEST_UPSTREAM}" + if "$here/mirror-index-copy.sh" "$u" "${REG}/${run}/up:1" >"$work/o5" 2>&1 && [ "$(digest "${REG}/${run}/up:1")" = "${u#*@}" ]; then ok "upstream index copied with identical digest ${u#*@}"; grep '^ok:' "$work/o5"; else bad "upstream copy"; tail -5 "$work/o5"; fi +fi +echo "passed=$pass failed=$failn"; [ "$failn" -eq 0 ] diff --git a/.gitea/workflows/mirror-base-image.yaml b/.gitea/workflows/mirror-base-image.yaml index 563e6bc..a8796e3 100644 --- a/.gitea/workflows/mirror-base-image.yaml +++ b/.gitea/workflows/mirror-base-image.yaml @@ -1,8 +1,8 @@ name: Mirror base images run-name: weekly base-image mirror -# Pulls each declared base image from upstream and re-pushes to the in-house -# registry, so any of our images that FROM these don't depend on docker.io's +# Copies each declared base image (the whole multi-arch INDEX, not one +# platform) from upstream to the in-house registry, so any of our images that FROM these don't depend on docker.io's # Cloudflare R2 blob storage being reachable. The 2026-05-12 Cloudflare # incident motivated this for python:3.12-slim and again for golang:1.25 # when the coraza-spoa build hit the same blob-fetch failure. @@ -55,6 +55,12 @@ jobs: echo "base-image: ${HUB_TAG} -> ${ref} (digest from registry-1.docker.io)" echo "ref=${ref}" >> "$GITHUB_OUTPUT" + # imagetools needs buildx; same setup as the build workflows. + - name: Set up Docker Buildx + uses: https://github.com/docker/setup-buildx-action@v3 + + # The registry token is written ONLY by this trusted x86 runner step + # (unchanged from before; no new secret exposure). - name: Login to in-house registry uses: docker/login-action@v3 with: @@ -62,7 +68,14 @@ jobs: username: ${{ secrets.CI_USER }} password: ${{ secrets.CI_TOKEN }} - - name: Pull by digest, retag, push ${{ matrix.image.hub }} + # Registry-to-registry copy of the WHOLE upstream index (all platforms, + # incl. linux/arm64). The old pull/tag/push re-published only the runner's + # amd64 manifest. mirror-index-copy.sh stages to -staging, asserts + # amd64+arm64 present and index digest == the Docker-Hub-reported digest, + # and only then moves the real tag, so a bad copy never replaces a good + # multi-arch mirror. Logic + tests: .gitea/scripts/mirror-index-copy.sh, + # .gitea/scripts/test-mirror-index-copy.sh. + - name: Copy index by digest, assert platforms + digest ${{ matrix.image.hub }} # Values reach the script via env, never via ${{ }} inside run:. env: REF: ${{ steps.base.outputs.ref }} @@ -74,28 +87,6 @@ jobs: [[ "$DST_PATH" =~ ^[a-z0-9][a-z0-9._/-]*$ && "$DST_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || { echo "::error::bad destination"; exit 1; } name="${REF%@*}"; digest="${REF#*@}" [[ "$name" == */* ]] || name="library/${name}" # official images - SRC="docker.io/${name}@${digest}" - DST="repo.anhonesthost.net/${DST_PATH}:${DST_TAG}" - - echo "::group::Pulling ${SRC}" - docker pull "${SRC}" - echo "::endgroup::" - - # The pulled image must be recorded under exactly the digest Docker - # Hub reported (docker refuses content that does not match it anyway). - repo_digests=$(docker image inspect "${SRC}" -f '{{range .RepoDigests}}{{println .}}{{end}}') - grep -qxF -e "docker.io/${name}@${digest}" -e "${name#library/}@${digest}" -e "${name}@${digest}" <<<"$repo_digests" \ - || { echo "::error::pulled image is not recorded under ${digest}"; exit 1; } - echo "upstream digest: ${SRC}" - - docker tag "${SRC}" "${DST}" - - echo "::group::Pushing ${DST}" - docker push "${DST}" - echo "::endgroup::" - - # The mirror tag's manifest digest (single-platform image as pushed); - # differs from the multi-arch index digest above by design. - dst_digests=$(docker image inspect "${DST}" -f '{{range .RepoDigests}}{{println .}}{{end}}') - DST_DIGEST=$(grep -m1 -F "repo.anhonesthost.net/${DST_PATH}@" <<<"$dst_digests" || true) - echo "mirror digest: ${DST_DIGEST:-}" + .gitea/scripts/mirror-index-copy.sh \ + "docker.io/${name}@${digest}" \ + "repo.anhonesthost.net/${DST_PATH}:${DST_TAG}"