diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index fecb4bf..407307d 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -1,9 +1,15 @@ name: HAProxy Manager Build and Push -run-name: ${{ gitea.actor }} pushed a change to main +run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }} on: push: branches: - main + # Release candidates: a push to release/* builds the SAME image but publishes + # ONLY :-rc and :, never :latest and never the plain : + # (see the "Resolve tags" step). Main keeps its behaviour. + - 'release/**' + tags: + - 'rc-*' # Manual publish (e.g. after a [skip ci] merge, or to re-publish after a # failed arm64 leg). workflow_dispatch: @@ -77,6 +83,30 @@ jobs: id: ver run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT" + # Publishing strategy. main: :latest + : + : (unchanged). + # release/* branch or rc-* tag: :-rc + : ONLY. A release candidate + # is deployed per host by explicit tag (durable per-host pin in WHP); :latest is + # moved to it by a later main publish only after the staged rollout succeeded. + # Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused + # rather than guessed at. + - name: Resolve tags + id: tags + env: + REF: ${{ gitea.ref }} + VERSION: ${{ steps.ver.outputs.version }} + SHA: ${{ gitea.sha }} + run: | + case "$REF" in + refs/heads/main) + tags="latest ${VERSION} ${SHA}" ;; + refs/heads/release/*|refs/tags/rc-*) + tags="${VERSION}-rc ${SHA}" ;; + *) + echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;; + esac + echo "tags=${tags}" >> "$GITHUB_OUTPUT" + echo "publishing tags: ${tags}" + # amd64: required. Pushed untagged by digest to both registries (same # digest in each); tags are only created by the assemble step below. - name: Build amd64 (push by digest) @@ -103,19 +133,79 @@ jobs: PKG_CACHE=${{ steps.pkgcache.outputs.url }} outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true - # Three tags per registry: :latest (moving), : (human-readable - # release), : (immutable, guaranteed-unique rollback target). - - name: Assemble tag indexes + # imagetools needs a NEW buildx. Same pinned, sha256-verified binary as + # mirror-base-image.yaml (kept identical on purpose). The assemble step + # previously ran on whatever buildx the (possibly ephemeral) runner had + # (0.13.1 on the runner that hung in run 5375, 0.15.1 on the trusted one). + - name: Install pinned buildx env: + BUILDX_VERSION: v0.28.0 + BUILDX_SHA256: 696bc104bac3bb708eff1af3f8bbc09fda0fd88f5757c1f9b404a35117889224 + run: | + set -euo pipefail + dir="${RUNNER_TEMP:-/tmp}/pinned-buildx"; mkdir -p "$dir" + curl -fsSL --retry 3 --max-time 120 -o "$dir/docker-buildx" \ + "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64" + echo "${BUILDX_SHA256} $dir/docker-buildx" | sha256sum -c - + chmod +x "$dir/docker-buildx" + "$dir/docker-buildx" version + echo "BUILDX=$dir/docker-buildx" >> "$GITHUB_ENV" + + # main: three tags per registry: :latest (moving), : (human-readable + # release), : (immutable, guaranteed-unique rollback target). + # release/* or rc-*: :-rc and : only (see "Resolve tags"). + # Hardened after run 5375 (assemble hung after pushing, TTY progress, old + # buildx): plain progress, pinned buildx, hard 10 minute step timeout. + - name: Assemble tag indexes + id: assemble + timeout-minutes: 10 + env: + BUILDKIT_PROGRESS: plain AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} ARM64_OUTCOME: ${{ steps.arm64.outcome }} VERSION: ${{ steps.ver.outputs.version }} - SHA: ${{ gitea.sha }} + TAGS: ${{ steps.tags.outputs.tags }} run: | - scripts/assemble-multiarch-index.sh \ + set -euo pipefail + tag_args=() + for t in $TAGS; do tag_args+=(--tag "$t"); done + # Use the pinned buildx for create AND the script's platform inspects. + export IMAGETOOLS="${BUILDX} imagetools" + out=$(scripts/assemble-multiarch-index.sh \ --amd64 "$AMD64_DIGEST" \ --arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \ --repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \ --repo ghcr.io/shadowdao/haproxy-manager-base \ - --tag latest --tag "$VERSION" --tag "$SHA" + "${tag_args[@]}" | tee /dev/stderr) + platforms=$(printf '%s\n' "$out" | sed -n 's/^platforms=//p' | tail -n1) + [ -n "$platforms" ] || { echo "::error::assemble script reported no platforms line"; exit 1; } + if [ "$platforms" = "linux/amd64" ]; then + echo "::warning::SINGLE-ARCH publish (amd64 only) on purpose: arm64 outcome was '${ARM64_OUTCOME}'. Tags carry no arm64 manifest." + else + echo "multi-arch publish: ${platforms}" + fi + echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + + # Final gate: every tag in every registry must resolve, carry the platforms + # the assemble step reported, and have ONE digest across all tags and both + # registries (they are built from the same sources). Any miss fails the job + # instead of trusting that the create calls "looked" fine. + - name: Verify published tags in every registry + timeout-minutes: 10 + env: + BUILDKIT_PROGRESS: plain + TAGS: ${{ steps.tags.outputs.tags }} + PLATFORMS: ${{ steps.assemble.outputs.platforms }} + run: | + set -euo pipefail + # Version-independent digest logic lives in the script (tested by + # scripts/test-verify-published-tags.py); buildx 0.28 ignores --format. + export IMAGETOOLS="${BUILDX} imagetools" + tag_args=() + for t in $TAGS; do tag_args+=(--tag "$t"); done + scripts/verify-published-tags.sh \ + --platforms "$PLATFORMS" \ + --repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \ + --repo ghcr.io/shadowdao/haproxy-manager-base \ + "${tag_args[@]}" diff --git a/CLAUDE.md b/CLAUDE.md index dff3ec4..2da050c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **WAF body inspection**: `python3 scripts/test-waf-body-inspection.py` - offline (runs in the image build); pins `option http-buffer-request`, the `X-WHP-WAF-Body` complete/partial stamp (strip -> stamp -> SPOE -> strip), `tune.bufsize` against the 100 ms SPOE fail-open budget and the 65535-byte SPOP frame cap, and Coraza's body limits in `coraza-spoa/config.yaml`. Run it after touching the Coraza block, `tune.bufsize`, or the SPOE engine. - **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired//` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`. - **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch"). +- **Publish verify gate**: `python3 scripts/test-verify-published-tags.py` - offline (needs `jq`; NOT run in the image build); `scripts/verify-published-tags.sh` is the last build-push.yaml step: every tag in every registry must resolve, carry the assembled platforms, and share ONE index digest. The digest is the sha256 of `imagetools inspect --raw` cross-checked against the text `Digest:` line / bare `--format` output (buildx 0.28 ignores `--format` and prints full text; CI run 5381). - **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash". +- **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines. - **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`. - **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md @@ -178,4 +180,5 @@ re-reads it on reload, so a failed runtime command must degrade to - **The :8000 API refuses peers other than 127.0.0.1/::1 and the container's default gateway** (`HAPROXY_MGR_API_ALLOWED_PEERS`, default `127.0.0.1,::1,gateway`; ports via `HAPROXY_MGR_PEER_FILTER_PORTS`, default `8000`). haproxy-manager shares `client-net` with every tenant container; host callers arrive from the gateway via docker-proxy. Only effective with the WHP `-p 127.0.0.1:8000:8000` publish (a 0.0.0.0 publish lets a tenant hairpin through the host and arrive from the gateway). A sidecar that must call :8000 directly needs its IP added - gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat - Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/` -- HAProxy is version 3.0.11 \ No newline at end of file +- HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101) +- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (dump blocks opened only by the anchored `Thread N is about to kill` header, `[ALERT] (PID) :`, `Current worker (N) exited with code N`; ANCHORED patterns, so keywords echoed from URLs/Host/server names never match) bypasses the bucket within a budget (1000 lines / 10 min, 60 s cooldown after a bound closes a block) and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7b03dbd..c0d8611 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,49 +23,78 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \ org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.licenses="MIT" -# haproxy is PINNED to a reviewed UPSTREAM version (HAPROXY_UPSTREAM), not to one -# Debian package revision. It was previously unpinned, so the binary could move -# under us at Debian's timing - an upstream release that rejected our config would -# have broken an unrelated commit's build, or worse, shipped an edge that refuses -# to start (see the `haproxy -c` gate below for why that matters: a config HAProxy -# rejects leaves the container Up with 80/443 unbound and /health still 200). +# haproxy comes from the UPSTREAM-maintained apt repo https://haproxy.debian.net +# (Vincent Bernat's builds, the Debian maintainer's own), suite +# `trixie-backports-`, NOT from Debian trixie's archive. # -# History: this used to pin the exact Debian version (3.0.11-1+deb13u3). Debian -# security then shipped 3.0.11-1+deb13u4 (same upstream 3.0.11, same arch set) and -# the exact pin silently froze the image on the OLDER, unpatched revision, and -# would hard-fail the build as soon as u3 left the archive. Debian security -# revisions of the SAME upstream version are exactly what we want to take, so: -# * an apt preference (Pin: version 3.0.11-*) makes apt install the NEWEST -# Debian revision of that upstream version on any architecture; -# * the build then FAILS unless the installed package AND `haproxy -v` both -# report that upstream version, so a major/minor bump (3.0 -> 3.1, 3.0.12) -# can never arrive by accident: that still needs this ARG changed and the -# `haproxy -c` gate below to pass. -# The pin and the gate compose: pinned = the version moves deliberately; the gate -# answers immediately whether the new binary still accepts fleet config. +# WHY (2026-11-16 forensics): Debian trixie ships 3.0.11 (+ security revisions). +# That build aborts via HAProxy's own watchdog inside the DNS resolver +# (resolv_process_responses / snr_resolution_cb, haproxy issue #3101) - whole +# edge down 15-52 s per abort on sdbees/whp01. The fix ("dns: bind the +# nameserver sockets to the initiating thread", "resolvers: make +# process_resolvers() single-threaded") is in 3.0.13+, 3.1.11+ and 3.2.6+; +# Debian trixie will not take it. See docs/go-port/haproxy-edge-upgrade-and-logging.md +# in the WHP repo for the version evidence and the rollout plan. # -# To move to another upstream version: change HAPROXY_UPSTREAM, rebuild, let the -# gate verify. Check what Debian offers with: apt-cache madison haproxy -ARG HAPROXY_UPSTREAM=3.0.11 +# Series: 3.2 (LTS, supported to 2030-Q2). Its trixie package is built with +# USE_QUIC=1 against the system OpenSSL 3.5.x NATIVE QUIC API (depends on +# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream +# has no trixie packages and would need a source build.) +# +# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 .. +# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any +# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian +# package version, HAPROXY_EXACT (default below), and the build: +# * pins apt to it (`Pin: version `; NOTE a bare +# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1", +# and with a non-matching pin apt falls back to Debian's own 3.0.11 package); +# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v` +# reports the series (so an unreachable repo, which would leave Debian's +# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident); +# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c` +# gate below proves the binary accepts the fleet config. +# Taking a new patch is a DELIBERATE edit of the default below (or +# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never +# passes it, so a rebuild is reproducible. +# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat, +# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with +# signed-by, never fetched at build time. The fingerprint is asserted below. +ARG HAPROXY_UPSTREAM=3.2 +ARG HAPROXY_EXACT=3.2.26-1~bpo13+1 # OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 # (CT130 pkg-cache, https only; CI sets it only after probing the cache with the # pinned CA). Unset = the upstream Debian repositories, exactly as before. The # apt sources are restored before this RUN ends (restore FAILS the build if any # reference to the cache is left), so no shipped layer carries the cache address. +# NOTE: CT130 does not proxy haproxy.debian.net (see the design doc); that one +# repo is always fetched directly. # Canonical script + docs: cloud-apache-container (scripts/pkg-cache-repos.sh, # docs/pkg-cache.md there); this repo carries byte-identical copies. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ +COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/haproxy-archive-keyring.gpg RUN /usr/local/sbin/pkg-cache-repos.sh apt && \ apt update -y && apt dist-upgrade -y && \ - printf 'Package: haproxy\nPin: version %s-*\nPin-Priority: 1001\n' "${HAPROXY_UPSTREAM}" > /etc/apt/preferences.d/haproxy-upstream-pin && \ + apt install -y --no-install-recommends gnupg ca-certificates && \ + keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \ + [ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \ + [ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \ + apt purge -y gnupg && \ + echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \ + apt update -y && \ + pin="${HAPROXY_EXACT}" && \ + printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \ apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \ rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \ deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \ - case "${deb_ver}" in "${HAPROXY_UPSTREAM}"-*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not upstream ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \ - haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}-" && \ - echo "haproxy pin OK: upstream ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \ + [ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \ + case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \ + haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \ + haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \ + if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \ + mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \ + echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \ /usr/local/sbin/pkg-cache-repos.sh restore && \ apt clean && rm -rf /var/lib/apt/lists/* WORKDIR /haproxy @@ -122,11 +151,14 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \ && python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \ && python3 /haproxy/scripts/test-default-splash.py \ && python3 /haproxy/scripts/test-access-log-receiver.py \ + && python3 /haproxy/scripts/test-haproxy-stderr-relay.py \ + && python3 /haproxy/scripts/test-haproxy-supervisor.py \ && python3 /haproxy/scripts/test-config-rollback.py \ && python3 /haproxy/scripts/test-cert-write-safety.py \ && python3 /haproxy/scripts/test-cert-scripts.py \ && python3 /haproxy/scripts/test-waf-body-inspection.py \ && python3 /haproxy/scripts/test-gunicorn-socket-timeouts.py \ + && python3 /haproxy/scripts/test-gunicorn-access-logging.py \ && python3 /haproxy/scripts/validate-rendered-config.py # Create log directories RUN mkdir -p /var/log/haproxy && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log diff --git a/README.md b/README.md index 1feb108..f1b0a1e 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,8 @@ Covered by `scripts/test-default-splash.py` (template order/conditions, the page, list generation) and the `splash-off` scenario of `scripts/validate-rendered-config.py`. +> gunicorn access logs (:8080 none, :8000 bounded file, not stdout): see docs/logging.md. + ## Access Log (in-container, since 2026.11.2) HAProxy logs (`log /log.sock ... format rfc5424`, `hap_header.tpl`) to a UNIX @@ -492,6 +494,22 @@ Rollout: image bump + recreate with the volume and sysctl (WHP's are in the image, outside the `/etc/haproxy` named volume, so nothing can be shadowed; `haproxy.cfg` is regenerated on start with the new `log` line. +## HAProxy stderr (panic dumps, [ALERT]s) - since 2026.11.4 + +HAProxy is started as `haproxy -W -db` so the master keeps a real stderr (with +the `daemon` keyword HAProxy points it at /dev/null, which discarded the +`[ALERT]` lines, `Current worker (N) exited with code 134` and the whole +watchdog/`ha_panic` thread dump). It is appended to +`/var/log/haproxy/haproxy-stderr.log` (the `haproxy-logs` volume, so it +survives a recreate) and `scripts/haproxy-stderr-relay.py` re-emits every line +on the container's stderr prefixed `haproxy-stderr: ` (rate-limited, file +rotated at 20 MiB x 3). `docker logs haproxy-manager` and whatever docker +logging driver the host configured therefore carry the crash text; the +per-request access log stays out of it. The in-container supervisor probes +with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` seconds (default 2, was 15) and +backs off to `HAPROXY_SUPERVISOR_MAX_BACKOFF` (30) only when a restart attempt +fails. Measured (3.2.26, empty config): crash to listeners back = 1.2-2.0 s. + ## Default Backend Configuration With the default splash on (above), unrouted hosts no longer reach this page; it still serves the blocked-IP and suspended pages, and serves unrouted hosts again when `HAPROXY_DEFAULT_SPLASH=off`. diff --git a/VERSION b/VERSION index 99b89ef..5b0a96a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.11.3 +2026.11.6 diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 0000000..ccbdade --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,43 @@ +# Container logging policy (what reaches `docker logs`) + +`docker logs haproxy-manager` (and the docker logging driver: json-file today, +journald -> Graylog under WHP PR #260) must carry only low-volume, high-value +lines. Per-request logs stay out of it. + +| Source | Destination | Volume | +|---|---|---| +| HAProxy edge access log (1.5M lines/day) | `/var/log/haproxy/access.log` via `haproxy-log-receiver.py` (20 MiB cap, gzip generations) | NOT in `docker logs` | +| HAProxy master stderr (ALERT, crash dumps) | `haproxy-stderr.log` -> `haproxy-stderr-relay.py` -> stdout (benign, rate limited) / stderr (crash evidence) | rare | +| Supervisor / relay / init lines | stdout / stderr | rare | +| gunicorn :8080 default-page / blocked-IP / suspended-site | NO access log | none | +| gunicorn :8000 management API access log | `/var/log/haproxy/api-access.log` (haproxy-logs volume, mode 0640), rotated by `scripts/gunicorn-access-rotate.py` (copy-truncate, 20 MiB x 3) | NOT in `docker logs` | +| gunicorn error log (tracebacks, worker timeouts) | stderr (`--error-logfile -`) | rare | + +## Why + +Before this change both gunicorns ran with `--access-logfile -`, so each :8080 +hit and each :8000 API call was one docker stdout line. With the journald +driver that is one journal entry per request: journald's default rate limit +(10000 messages / 30 s per sending service) can then suppress the very +crash-dump lines the relay emits, and Graylog would be flooded. + +## Knobs + +`HAPROXY_MGR_API_ACCESS_LOG` (path), `HAPROXY_API_ACCESS_LOG_MAX_BYTES` +(20971520), `HAPROXY_API_ACCESS_LOG_BACKUPS` (3), `HAPROXY_API_ACCESS_LOG_POLL` +(30 s; the live file can exceed the cap by at most one interval of API traffic). + +## Not done + +Failed-auth (401/403) lines are NOT mirrored to stdout; they are in the file +(grep the status column). Add a rate-limited mirror later if wanted. + +## Tests + +`python3 scripts/test-gunicorn-access-logging.py` (runs in the image build): +real gunicorn, a burst of default-page requests produces zero stdout lines, +API requests land in the file not stdout, handler exceptions still reach +stderr, and rotation is bounded. + +Deploying this needs a NEW rc image built from `feat/haproxy-upgrade` +(VERSION stays 2026.11.5 until then). diff --git a/haproxy_manager.py b/haproxy_manager.py index 395c25d..7465ce1 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -766,6 +766,92 @@ def is_process_running(process_name): return True return False +# --------------------------------------------------------------------------- +# Launching the HAProxy master (2026-11 forensics change) +# +# haproxy.cfg carries `daemon`, and in daemon mode HAProxy re-points the +# master's stdin/stdout/stderr at /dev/null after it has parsed the config +# (MEASURED on 3.0.11: /proc//fd/2 -> /dev/null). That discarded every +# [ALERT], the "exited with code 134" master line and the whole ha_panic()/ +# watchdog thread dump of the 2026-11 aborts: only the core file survived. +# +# `-db` overrides `daemon` (MEASURED: foreground master, still master-worker, +# reload via the CLI keeps the same master pid), so the master keeps a real +# stderr. We append it to HAPROXY_STDERR_LOG (haproxy-logs volume, survives a +# container recreate) and scripts/haproxy-stderr-relay.py forwards it to the +# container's stderr (-> docker logging driver -> host journald/Graylog). +# +# The master is started as `sh -c '... &'` so it is re-parented to PID 1 exactly +# as the old daemon fork did (PID 1 reaps it; a Popen child would sit as a +# zombie in the API worker and is_process_running('haproxy') would keep +# reporting it alive). +# --------------------------------------------------------------------------- +HAPROXY_STDERR_LOG = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log') +try: + HAPROXY_START_WAIT = float(os.environ.get('HAPROXY_START_WAIT', '10')) +except ValueError: + HAPROXY_START_WAIT = 10.0 + + +def _stderr_log_path(): + """HAPROXY_STDERR_LOG if its directory is writable, else a /tmp fallback + (a missing volume must never stop HAProxy from starting).""" + path = HAPROXY_STDERR_LOG + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + # 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable) + os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640)) + return path + except OSError: + return '/tmp/haproxy-stderr.log' + + +def spawn_haproxy_master(): + """Start `haproxy -W -db` detached, stderr appended to the stderr log. + + Returns a subprocess.CompletedProcess (returncode 0, .stderr = the new log + text) once HAProxy has loaded its configuration; raises + subprocess.CalledProcessError (.stderr = what HAProxy printed) if it exits + or never comes up within HAPROXY_START_WAIT seconds. Same contract the old + blocking `subprocess.run([... '-W' ...], check=True)` call sites relied on. + """ + cmd = ['haproxy', '-W', '-db', '-S', '/tmp/haproxy-cli,level,admin', + '-f', HAPROXY_CONFIG_PATH] + log_path = _stderr_log_path() + try: + offset = os.path.getsize(log_path) + except OSError: + offset = 0 + subprocess.run( + ['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; ' + '"$@" >"$log" 2>&1 &', 'sh', log_path] + cmd, + check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + def new_text(): + try: + with open(log_path, 'rb') as fh: + fh.seek(offset) + return fh.read().decode('utf-8', 'replace') + except OSError: + return '' + + deadline = time.monotonic() + HAPROXY_START_WAIT + started = time.monotonic() + while time.monotonic() < deadline: + text = new_text() + if 'Loading success.' in text: + return subprocess.CompletedProcess(cmd, 0, '', text) + if time.monotonic() - started > 1.0 and not is_process_running('haproxy'): + break + time.sleep(0.1) + text = new_text() + if is_process_running('haproxy') and '[ALERT]' not in text: + # Slow to log (big config) but alive: treat as started. + return subprocess.CompletedProcess(cmd, 0, '', text) + raise subprocess.CalledProcessError(1, cmd, output='', stderr=text[-4000:]) + + # Initialize template engine template_loader = jinja2.FileSystemLoader(TEMPLATE_DIR) template_env = jinja2.Environment(loader=template_loader) @@ -871,12 +957,7 @@ def reload_haproxy(): return jsonify({'status': 'success'}), 200 else: # Start HAProxy if it's not running - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, - capture_output=True, - text=True - ) + result = spawn_haproxy_master() if result.returncode == 0: print("HAProxy started successfully") log_operation('start_haproxy', True) @@ -3778,10 +3859,7 @@ def reload_haproxy_safely(backup_status=None): else: # HAProxy not running, start it try: - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, capture_output=True, text=True - ) + result = spawn_haproxy_master() logger.info("HAProxy started successfully") # Now - and only now - is this config known good. promote_current_config_to_backup() @@ -4114,12 +4192,7 @@ def start_haproxy(): return # Configuration is valid, start HAProxy - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, - capture_output=True, - text=True - ) + result = spawn_haproxy_master() if result.returncode == 0: logger.info("HAProxy started successfully") log_operation('start_haproxy', True, 'HAProxy started successfully') diff --git a/keys/haproxy-debian-net/haproxy-archive-keyring.gpg b/keys/haproxy-debian-net/haproxy-archive-keyring.gpg new file mode 100644 index 0000000..17b3fe3 Binary files /dev/null and b/keys/haproxy-debian-net/haproxy-archive-keyring.gpg differ diff --git a/scripts/ensure_haproxy.py b/scripts/ensure_haproxy.py index a91755e..ff3107d 100644 --- a/scripts/ensure_haproxy.py +++ b/scripts/ensure_haproxy.py @@ -34,6 +34,18 @@ def main(): haproxy_manager.logger.warning( "[haproxy-supervisor] haproxy process not found — attempting in-place restart" ) + # Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said + # before it vanished (the relay forwards the same text; this ties it to the + # supervisor event even if the relay is behind). + try: + with open(haproxy_manager._stderr_log_path(), 'rb') as fh: + fh.seek(0, 2) + fh.seek(max(0, fh.tell() - 1500)) + tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:] + for line in tail: + haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line) + except OSError: + pass # start_haproxy() validates the config (and regenerates it if invalid) # before launching, and swallows its own errors, so it will not raise here. haproxy_manager.start_haproxy() diff --git a/scripts/gunicorn-access-rotate.py b/scripts/gunicorn-access-rotate.py new file mode 100755 index 0000000..57518ae --- /dev/null +++ b/scripts/gunicorn-access-rotate.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Bound the management-API (:8000) gunicorn access log file. + +Why +--- +Gunicorn's access log used to go to stdout (`--access-logfile -`), i.e. every +:8080 default-page / blocked-IP / suspended-site hit and every :8000 API call +became a `docker logs` line. With the journald docker log driver that is one +journal entry per request (journald's default rate limit, 10000 msgs / 30 s per +sending service, can then SUPPRESS the crash-dump lines the relay emits) and it +floods Graylog. Now: + * :8080 (default page) has NO access log at all; + * :8000 (API) writes its access log to API_ACCESS_LOG (default + /var/log/haproxy/api-access.log, in the haproxy-logs volume), NOT stdout; + * gunicorn ERROR logging (tracebacks, worker timeouts) stays on stderr. +This process keeps that file bounded: copy-truncate rotation at +HAPROXY_API_ACCESS_LOG_MAX_BYTES (default 20 MiB), HAPROXY_API_ACCESS_LOG_BACKUPS +generations (default 3), rotated copies mode 0640 (same policy as +haproxy-stderr.log). Truncating in place is safe because gunicorn opens the +file in append mode. Several gunicorn processes share the file, so renaming it +is not an option. + +Bound: size is checked every HAPROXY_API_ACCESS_LOG_POLL seconds (default 30), +so the live file can exceed the limit only by what the API writes in one +interval (a few hundred bytes per call). +""" +import os +import shutil +import sys +import time + + +def _env_num(name, default, cast=int): + try: + v = cast(os.environ.get(name, default)) + return v if v > 0 else default + except (TypeError, ValueError): + return default + + +def rotate_if_needed(path, max_bytes, backups): + """Copy-truncate rotate. Returns True if it rotated.""" + try: + if os.path.getsize(path) <= max_bytes: + return False + for i in range(backups, 1, -1): + src = '%s.%d' % (path, i - 1) + if os.path.exists(src): + os.replace(src, '%s.%d' % (path, i)) + shutil.copyfile(path, path + '.1') + os.chmod(path + '.1', 0o640) + with open(path, 'r+b') as fh: + fh.truncate(0) + return True + except FileNotFoundError: + return False + except OSError as exc: + sys.stderr.write('gunicorn-access-rotate: rotate failed: %s\n' % exc) + return False + + +def main(): + path = os.environ.get('HAPROXY_MGR_API_ACCESS_LOG', '/var/log/haproxy/api-access.log') + max_bytes = _env_num('HAPROXY_API_ACCESS_LOG_MAX_BYTES', 20 * 1024 * 1024) + backups = _env_num('HAPROXY_API_ACCESS_LOG_BACKUPS', 3) + poll = _env_num('HAPROXY_API_ACCESS_LOG_POLL', 30.0, float) + while True: + try: + rotate_if_needed(path, max_bytes, backups) + except Exception as exc: # never die; the supervisor loop restarts us anyway + sys.stderr.write('gunicorn-access-rotate: %r\n' % exc) + time.sleep(poll) + + +if __name__ == '__main__': + main() diff --git a/scripts/haproxy-stderr-relay.py b/scripts/haproxy-stderr-relay.py new file mode 100755 index 0000000..f5119b9 --- /dev/null +++ b/scripts/haproxy-stderr-relay.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Forward HAProxy's own stderr to the container's stderr, and bound the file. + +Why +--- +HAProxy's master stderr carries what the access log never will: `[ALERT]` +lines, "Current worker (N) exited with code 134", and the complete +ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master() +launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the +haproxy-logs volume, so it survives a container recreate). This process tails the +file and re-emits each line on ITS stderr, prefixed, which is the container's +stderr: `docker logs`, and through the docker logging driver the host journal +and Graylog. + +Properties +---------- +* Rare by nature: HAProxy only writes here on start/reload/server state + changes/crashes. The per-request access log is NOT here (it goes to the + receiver socket), so this does not feed the WHP AI monitor a firehose. +* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE + ("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is + about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/ + server name never matches) and every line of a dump block (from a panic/watchdog header to + the master's "All workers exited" line, bounded by + DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block + can open; blocks open only on the anchored "Thread N is about to kill" header) + are forwarded in full even when the bucket is empty, within a total budget of + CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter + and are counted in the suppressed summary). They go to STDERR. +* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!", + "Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints + the first at [ALERT]: every tenant container stop/crash/deploy makes them. + Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own + "[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI + monitor through `docker logs haproxy-manager`; rate-limiting them is intended. + Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend. +* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)", + config "parsing [...]" warnings, "Former worker (N) exited with code 0", + "Initializing new worker", "Loading success", ...) never touches the bucket: + it is counted and collapsed into ONE "[relay] reload/start ok: N routine + lines collapsed" line per reload (plus one on the minute tick for the late + "Proxy stopped" lines). The full text stays in the stderr file. +* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST + lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute + "[relay] suppressed N lines" summary. +* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and + stderr -> PRIORITY 3 (err) and does NOT parse "" prefixes. So benign + lines (routine, rate-limited remainder, relay summaries) are written to + STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority. +* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES + (default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3). + Truncating is safe because HAProxy's fd is O_APPEND. +* Never blocks or affects HAProxy: it only reads a file. + +Environment +----------- + HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120), + HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3), + HAPROXY_STDERR_RELAY_POLL (0.5 s) +""" +import os +import re +import shutil +import sys +import time + +PREFIX = 'haproxy-stderr: ' +MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output) +MAX_PENDING = 65536 # an unterminated line is force-emitted past this size +DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this +DUMP_MAX_SECONDS = 30.0 +SUMMARY_EVERY = 60.0 + +# Every pattern is ANCHORED to HAProxy's own message structure (captured from +# real 3.2 crashes): a keyword inside a request path / Host header / server +# name that HAProxy echoes into a warning can never match, because the line +# prefix "[ALERT] (PID) :" is written by HAProxy itself. +# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET. +CRASH_RE = re.compile( + r'^(?:' + r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)' + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+' + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died' + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b' + r'|Thread \d+ is about to kill the process' + r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ ' + r')') +# Tenant-driven state lines. WHP gives every tenant server `check`, so every +# tenant container stop/crash/OOM/deploy and every host-wide container restart +# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash +# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so +# they can neither reach priority 3 nor spend the general bucket / crash budgets. +TENANT_RE = re.compile( + r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:' + r"backend '[^']*' has no server available!" + r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)' + r'|Health check for (?:backup )?server \S+ ' + r')') +# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE. +DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process') +DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b') +# A new start/reload cycle also closes a dump block. +CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)') +# Routine per-start/reload noise: counted and collapsed, never rate-limited. +ROUTINE_RE = re.compile( + r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:' + r'Proxy \S+ stopped \(cumulated conns' + r'|config\s*:\s*parsing \[[^\]]*\]\s*:' + r'|Former worker \(\d+\) exited with code 0\b' + r'|Reloading HAProxy|Initializing new worker|Loading success' + r'|path to executable is|haproxy version is' + r"|config: Can't load stats-file|'option forwardfor' ignored)") +LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success') +# Total exemption budget: even real-looking evidence cannot flood without bound. +CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70) +CRASH_WINDOW = 600.0 # seconds +DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that + # ordinary evidence lines (worker exits, alerts) can never spend +TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute) +TENANT_PER_MIN = 30 +DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long + + +def _env_num(name, default, cast=int): + try: + v = cast(os.environ.get(name, default)) + return v if v > 0 else default + except (TypeError, ValueError): + return default + + +class Relay: + def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None): + self.path = path + self.out = out # benign lines (docker journald: PRIORITY 6) + self.err = err if err is not None else out # crash lines (PRIORITY 3) + self.burst, self.per_min = burst, per_min + self.max_bytes, self.backups = max_bytes, backups + self.clock = clock + self.tokens = float(burst) + self.last = clock() + self.suppressed = 0 + self.routine = 0 # collapsed routine lines since the last summary + self.summary_at = clock() + self.in_dump = False + self.dump_lines = 0 + self.dump_since = 0.0 + self.cooldown_until = 0.0 + self.budget_left = CRASH_BUDGET + self.window_start = clock() + self.dump_budget_left = DUMP_BUDGET + self.dump_window_start = clock() + self.tenant_tokens = float(TENANT_BURST) + self.tenant_last = clock() + self.tenant_suppressed = 0 + self.offset = None # None = first pass: start at EOF (do not replay history) + self.pending = '' + self._last_stream = None + + def _take(self): + now = self.clock() + self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0) + self.last = now + if self.tokens >= 1: + self.tokens -= 1 + return True + return False + + def _take_tenant(self): + now = self.clock() + self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0) + self.tenant_last = now + if self.tenant_tokens >= 1: + self.tenant_tokens -= 1 + return True + return False + + def _write(self, stream, text): + # keep the two streams in emission order: flush the other one first + if self._last_stream is not None and self._last_stream is not stream: + self._last_stream.flush() + stream.write(PREFIX + text + '\n') + self._last_stream = stream + + def _flush_all(self): + for st in {id(self.out): self.out, id(self.err): self.err}.values(): + try: + st.flush() + except Exception: + pass + + def _summaries(self, force=False): + """Per-minute summary of what the limiter / collapse withheld.""" + now = self.clock() + if not force and now - self.summary_at < SUMMARY_EVERY: + return + self.summary_at = now + if self.routine: + self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)' + % self.routine) + self.routine = 0 + if self.tenant_suppressed: + self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines ' + '(rate limit; full text in the stderr file)' % self.tenant_suppressed) + self.tenant_suppressed = 0 + if self.suppressed: + self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)' + % self.suppressed) + self.suppressed = 0 + + def _budget(self, now, kind='crash'): + """Take one unit of a budget; False once the window's budget is spent. + + kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget).""" + if kind == 'dump': + if now - self.dump_window_start >= CRASH_WINDOW: + self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET + if self.dump_budget_left > 0: + self.dump_budget_left -= 1 + return True + return False + if now - self.window_start >= CRASH_WINDOW: + self.window_start, self.budget_left = now, CRASH_BUDGET + if self.budget_left > 0: + self.budget_left -= 1 + return True + return False + + def _end_dump(self, now, why=None): + self.in_dump = False + if why: # closed by a bound, not by the master's end line: no re-arming for a while + self.cooldown_until = now + DUMP_COOLDOWN + self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why) + + def _emit(self, line): + now = self.clock() + if self.in_dump: + if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS): + self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since)) + elif CYCLE_RE.match(line): + self._end_dump(now) + if self.in_dump or CRASH_RE.match(line): + is_dump = self.in_dump or bool(DUMP_START_RE.match(line)) + kind = 'dump' if is_dump else 'crash' + if self._budget(now, kind): + # crash evidence: bypasses the bucket, the collapse and the summary + if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until: + self.in_dump, self.dump_lines, self.dump_since = True, 0, now + if self.in_dump: + self.dump_lines += 1 + self._write(self.err, line) + if self.in_dump and DUMP_END_RE.match(line): + self._end_dump(now) + return + if self.in_dump: # budget spent mid-block: the normal limiter takes over + self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW)) + if ROUTINE_RE.match(line): + self.routine += 1 + if LOADED_RE.match(line): # one summary per start/reload + self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)' + % self.routine) + self.routine = 0 + self._summaries() + return + if TENANT_RE.match(line): + if self._take_tenant(): + self._write(self.out, line) + else: + self.tenant_suppressed += 1 + self._summaries() + return + if self._take(): + self._write(self.out, line) + else: + self.suppressed += 1 + self._summaries() + + @staticmethod + def _clean(line): + line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip() + if len(line) > MAX_LINE: + line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line) + return line + + def rotate_if_needed(self): + try: + if os.path.getsize(self.path) <= self.max_bytes: + return + for i in range(self.backups, 1, -1): + src = '%s.%d' % (self.path, i - 1) + if os.path.exists(src): + os.replace(src, '%s.%d' % (self.path, i)) + shutil.copyfile(self.path, self.path + '.1') + os.chmod(self.path + '.1', 0o640) + with open(self.path, 'r+b') as fh: + fh.truncate(0) + self.offset = 0 + except OSError as exc: + sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc) + + def poll_once(self): + try: + size = os.path.getsize(self.path) + except OSError: + if self.offset is None: + self.offset = 0 # file not created yet: whatever appears is new + return + if self.offset is None: + self.offset = size + return + if size < self.offset: # truncated by someone else + self.offset = 0 + if size == self.offset: + return + with open(self.path, 'rb') as fh: + fh.seek(self.offset) + chunk = fh.read(1 << 20) + self.offset += len(chunk) + text = self.pending + chunk.decode('utf-8', 'replace') + lines = text.split('\n') + self.pending = lines.pop() # incomplete trailing line stays buffered + if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line + lines.append(self.pending) + self.pending = '' + for line in lines: + line = self._clean(line) + if line: + self._emit(line) + self._summaries() + self._flush_all() + + def tick(self): + """Idle-time housekeeping: emit the per-minute summaries even when no line arrives.""" + self._summaries() + self._flush_all() + + +def main(): + path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log') + relay = Relay(path, sys.stdout, + _env_num('HAPROXY_STDERR_RELAY_BURST', 400), + _env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120), + _env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024), + _env_num('HAPROXY_STDERR_LOG_BACKUPS', 3), + err=sys.stderr) + poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float) + sys.stdout.write('haproxy-stderr-relay: following %s\n' % path) + sys.stdout.flush() + ticks = 0 + while True: + try: + relay.poll_once() + relay.tick() + ticks += 1 + if ticks % 120 == 0: + relay.rotate_if_needed() + except Exception as exc: # never die: the supervisor loop restarts us anyway + sys.stderr.write('haproxy-stderr-relay: %r\n' % exc) + time.sleep(poll) + + +if __name__ == '__main__': + main() diff --git a/scripts/haproxy-supervisor.sh b/scripts/haproxy-supervisor.sh new file mode 100755 index 0000000..f832942 --- /dev/null +++ b/scripts/haproxy-supervisor.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# In-container haproxy supervisor (started in the background by start-up.sh). +# +# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching +# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134, +# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart +# policy never fires, and haproxy is down until the external host watchdog +# full-restarts the whole container minutes later (dropping every connection). +# This loop revives haproxy in place within one interval. +# +# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports +# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and +# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone. +# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s, +# most of it detection latency + start; the default is now 2 s (was 15). +# +# Two backoffs: +# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its +# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of +# re-rendering/re-validating the same broken config every 2 s; +# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH +# seconds (default 10) of that start (a crash loop): extra wait before the +# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default +# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives +# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death +# after a long run is still revived within one interval. +# +# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5, +# "abc") falls back to the default instead of killing this subshell under set -e. +set -eo pipefail + +_posint() { # _posint VALUE DEFAULT + case "$1" in + ''|*[!0-9]*) echo "$2" ;; + *) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;; + esac +} + +INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)" +MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)" +QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)" +STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)" +STEPS="" +for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do + v="$(_posint "$s" 0)" + [ "$v" -ge 1 ] && STEPS="${STEPS} ${v}" +done +[ -n "${STEPS}" ] || STEPS=" 2 5 10 30" + +# step N (1-based, past the end = last), capped at MAX_BACKOFF +_step() { + local n="$1" i=0 v=0 + for s in ${STEPS}; do + i=$((i + 1)); v="$s" + [ "${i}" -ge "${n}" ] && break + done + [ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}" + echo "${v}" +} + +delay="${INTERVAL}" +last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet +quick=0 # consecutive deaths within QUICK_DEATH of a start +while true; do + sleep "${delay}" + if pidof -q haproxy; then + delay="${INTERVAL}" + if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then + quick=0; last_start=-1 + fi + continue + fi + if [ "${last_start}" -ge 0 ]; then + up=$((SECONDS - last_start)) + if [ "${up}" -lt "${QUICK_DEATH}" ]; then + quick=$((quick + 1)) + wait_s="$(_step "${quick}")" + echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2 + sleep "${wait_s}" + pidof -q haproxy && continue + elif [ "${up}" -ge "${STABLE}" ]; then + quick=0 + fi + fi + if python /haproxy/scripts/ensure_haproxy.py 2>&1; then + delay="${INTERVAL}" + last_start="${SECONDS}" + else + delay=$(( delay * 2 )) + [ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}" + fi +done diff --git a/scripts/start-up.sh b/scripts/start-up.sh index ba0c403..76bdfeb 100755 --- a/scripts/start-up.sh +++ b/scripts/start-up.sh @@ -75,24 +75,49 @@ mkdir -p /var/log/haproxy /var/lib/haproxy cron & +# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and +# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs, +# "exited with code 134", the watchdog/ha_panic thread dump) is appended to +# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's +# stderr so `docker logs` / the docker logging driver / Graylog get it. Started +# BEFORE init.py so the first start's lines are followed. Restarted within 2 s +# if it ever dies; it only reads a file, so it can never affect HAProxy. +( + while true; do + python /haproxy/scripts/haproxy-stderr-relay.py || true + sleep 2 + done +) & + +# gunicorn access logs (incident-prevention for the journald log driver). +# gunicorn used to log every request to stdout (`--access-logfile -`), so every +# :8080 default-page / blocked-IP hit and every :8000 API call became a docker +# log line -> journal entry -> Graylog line, and could trip journald's rate +# limit and suppress the relay's crash-dump lines. Now: +# :8080 no access log at all; +# :8000 access log to a size-bounded FILE in the haproxy-logs volume (mode +# 0640), rotated by scripts/gunicorn-access-rotate.py (copy-truncate, +# 20 MiB x 3). Set HAPROXY_MGR_API_ACCESS_LOG to relocate it. +# Gunicorn ERROR logging (--error-logfile -) stays on stderr -> `docker logs`. +API_ACCESS_LOG="${HAPROXY_MGR_API_ACCESS_LOG:-/var/log/haproxy/api-access.log}" +export HAPROXY_MGR_API_ACCESS_LOG="${API_ACCESS_LOG}" +mkdir -p "$(dirname "${API_ACCESS_LOG}")" +[ -e "${API_ACCESS_LOG}" ] || : > "${API_ACCESS_LOG}" +chmod 0640 "${API_ACCESS_LOG}" 2>/dev/null || true +( + while true; do + python /haproxy/scripts/gunicorn-access-rotate.py || true + sleep 2 + done +) & + # Phase 1: container init python /haproxy/scripts/init.py -# Phase 1.5: in-container haproxy supervisor. -# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching -# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134, -# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart -# policy never fires, and haproxy is down until the external host watchdog -# full-restarts the whole container minutes later (dropping every connection). -# This loop revives haproxy in place within one interval. ensure_haproxy.py is -# idempotent — a cheap no-op whenever haproxy is already running. -HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}" -( - while true; do - sleep "${HAPROXY_SUPERVISOR_INTERVAL}" - python /haproxy/scripts/ensure_haproxy.py 2>&1 || true - done -) & +# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh: +# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with +# backoff on failed restarts and on crash loops; knobs validated as integers). +/haproxy/scripts/haproxy-supervisor.sh & # Phase 2: WSGI servers # Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT @@ -132,7 +157,7 @@ gunicorn \ --workers 1 --threads 4 --worker-class "${WORKER_CLASS}" \ --max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \ --timeout 30 \ - --access-logfile - --error-logfile - --log-level info \ + --error-logfile - --log-level info \ --pythonpath /haproxy \ 'haproxy_manager:default_app' & @@ -144,6 +169,6 @@ exec gunicorn \ --workers "${API_WORKERS}" --threads 4 --worker-class "${WORKER_CLASS}" \ --max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \ --timeout "${API_TIMEOUT}" \ - --access-logfile - --error-logfile - --log-level info \ + --access-logfile "${API_ACCESS_LOG}" --error-logfile - --log-level info \ --pythonpath /haproxy \ 'haproxy_manager:app' diff --git a/scripts/test-cert-write-safety.py b/scripts/test-cert-write-safety.py index c89e742..ba582f9 100755 --- a/scripts/test-cert-write-safety.py +++ b/scripts/test-cert-write-safety.py @@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR) # The module opens /var/log/haproxy-manager.log at import time via # logging.FileHandler. Redirect that one call so the suite runs unprivileged. _LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +# spawn_haproxy_master() appends the master's stderr here. +os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log')) +os.environ.setdefault('HAPROXY_START_WAIT', '3') _real_file_handler = logging.FileHandler logging.FileHandler = ( lambda fn, *a, **kw: _real_file_handler( @@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ # haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN} # reject any unusable file in $TEST_CERTS_DIR cfg="" + start=0 while [ $# -gt 0 ]; do - case "$1" in -f) cfg="$2"; shift ;; esac + case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac shift done if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then @@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ fi done fi + # real `haproxy -W -db` prints this once the config is loaded + [ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2 exit 0 """) diff --git a/scripts/test-config-rollback.py b/scripts/test-config-rollback.py index 1b50725..6bcd730 100755 --- a/scripts/test-config-rollback.py +++ b/scripts/test-config-rollback.py @@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR) # The module opens /var/log/haproxy-manager.log at import time via # logging.FileHandler. Redirect that one call so the suite runs unprivileged. _LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +# spawn_haproxy_master() appends the master's stderr here (default is the +# haproxy-logs volume path, not writable unprivileged). +os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log')) +os.environ.setdefault('HAPROXY_START_WAIT', '3') _real_file_handler = logging.FileHandler logging.FileHandler = ( lambda fn, *a, **kw: _real_file_handler( @@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ # haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0 # haproxy -W -S ... -f FILE (start) -> same validation, then exit 0 cfg="" + start=0 while [ $# -gt 0 ]; do - case "$1" in -f) cfg="$2"; shift ;; esac + case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac shift done if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2 exit 1 fi + # A real `haproxy -W -db` prints this once the config is loaded; + # spawn_haproxy_master() waits for it. + [ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2 exit 0 """) diff --git a/scripts/test-gunicorn-access-logging.py b/scripts/test-gunicorn-access-logging.py new file mode 100755 index 0000000..92df42a --- /dev/null +++ b/scripts/test-gunicorn-access-logging.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Tests: gunicorn access logging must not flood the container's stdout. + +Context: with the journald docker log driver every stdout line is a journal +entry (default rate limit 10000 msgs/30 s per service could suppress crash-dump +lines) and, via Graylog, a stored message. start-up.sh therefore gives :8080 no +access log and sends :8000's to a bounded FILE; gunicorn errors stay on stderr. + +Pins (static, from start-up.sh): + * no gunicorn invocation uses `--access-logfile -`; + * the :8080 invocation has no --access-logfile at all; + * the :8000 invocation logs access to ${API_ACCESS_LOG}, errors to `-`. +Pins (live, REAL gunicorn, same flag shapes): + * a burst of default-page requests (no access log) writes ZERO lines to + stdout and none to stderr; + * a burst of API requests writes ZERO stdout lines, the access file receives + them, and a handler exception still reaches STDERR (error logging intact); + * scripts/gunicorn-access-rotate.py keeps the file bounded (copy-truncate, + backups capped, rotated copy mode 0640). +Runs in the Docker build gate. +""" +import importlib.util +import os +import re +import socket +import stat +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) + +try: + import gunicorn # noqa: F401 +except ImportError: + sys.stderr.write('SKIP: gunicorn not installed\n') + raise SystemExit(77) + +spec = importlib.util.spec_from_file_location('rot', os.path.join(HERE, 'gunicorn-access-rotate.py')) +rot = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rot) + +APP = textwrap.dedent('''\ + def app(environ, start_response): + if environ['PATH_INFO'] == '/boom': + raise RuntimeError('handler-exploded') + start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '2')]) + return [b'ok'] +''') + + +def free_port(): + s = socket.socket() + s.bind(('127.0.0.1', 0)) + p = s.getsockname()[1] + s.close() + return p + + +def start_script(): + with open(os.path.join(HERE, 'start-up.sh')) as fh: + return fh.read() + + +class StaticFlags(unittest.TestCase): + def setUp(self): + self.src = start_script() + # Each invocation = from "gunicorn \" to its app target line. + self.inv = re.findall(r'^(?:exec )?gunicorn \\\n(?:.*\\\n)*?.*haproxy_manager:\w+.*$', self.src, re.M) + + def test_two_invocations(self): + self.assertEqual(len(self.inv), 2, self.inv) + + def test_no_stdout_access_log(self): + for i in self.inv: + self.assertNotRegex(i, r'--access-logfile\s+-(\s|$)') + + def test_default_page_has_no_access_log(self): + d = [i for i in self.inv if '0.0.0.0:8080' in i] + self.assertEqual(len(d), 1) + self.assertNotIn('--access-logfile', d[0]) + self.assertIn('--error-logfile -', d[0]) + + def test_api_access_log_is_a_file(self): + a = [i for i in self.inv if '0.0.0.0:8000' in i] + self.assertEqual(len(a), 1) + self.assertIn('--access-logfile "${API_ACCESS_LOG}"', a[0]) + self.assertIn('--error-logfile -', a[0]) + + def test_rotator_started(self): + self.assertIn('gunicorn-access-rotate.py', self.src) + + +class Live(unittest.TestCase): + N_DEFAULT = 300 + N_API = 100 + + def setUp(self): + self.dir = tempfile.mkdtemp(prefix='gunicorn-access-') + with open(os.path.join(self.dir, 'tapp.py'), 'w') as fh: + fh.write(APP) + + def run_gunicorn(self, extra, n, boom=False): + port = free_port() + out = open(os.path.join(self.dir, 'out-%d' % port), 'w+') + err = open(os.path.join(self.dir, 'err-%d' % port), 'w+') + p = subprocess.Popen( + [sys.executable, '-m', 'gunicorn', '--bind', '127.0.0.1:%d' % port, + '--workers', '1', '--threads', '4', '--error-logfile', '-', '--log-level', 'info', + '--pythonpath', self.dir] + extra + ['tapp:app'], + stdout=out, stderr=err) + try: + for _ in range(100): + try: + urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=2).read() + break + except Exception: + time.sleep(0.2) + for _ in range(n): + urllib.request.urlopen('http://127.0.0.1:%d/x' % port, timeout=5).read() + if boom: + try: + urllib.request.urlopen('http://127.0.0.1:%d/boom' % port, timeout=5).read() + except Exception: + pass + time.sleep(0.5) + finally: + p.terminate() + p.wait(10) + out.seek(0) + err.seek(0) + o, e = out.read(), err.read() + out.close() + err.close() + return o, e + + def test_default_page_burst_is_silent(self): + out, err = self.run_gunicorn([], self.N_DEFAULT) + self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200]) + self.assertNotIn('GET /x', err) + + def test_api_burst_goes_to_file_not_stdout_and_errors_stay_on_stderr(self): + path = os.path.join(self.dir, 'api-access.log') + open(path, 'w').close() + out, err = self.run_gunicorn(['--access-logfile', path], self.N_API, boom=True) + self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200]) + self.assertNotIn('GET /x', err) + with open(path) as fh: + lines = fh.read().splitlines() + self.assertGreaterEqual(len([l for l in lines if 'GET /x' in l]), self.N_API) + self.assertIn('handler-exploded', err) # error logging intact + + +class Rotation(unittest.TestCase): + def test_bounded_and_mode(self): + d = tempfile.mkdtemp(prefix='gunicorn-rot-') + path = os.path.join(d, 'api-access.log') + for _ in range(12): + with open(path, 'ab') as fh: + fh.write(b'x' * 1000 + b'\n') + rot.rotate_if_needed(path, 4000, 3) + files = sorted(os.listdir(d)) + self.assertLessEqual(len(files), 4, files) # live + 3 backups + self.assertLessEqual(os.path.getsize(path), 4000 + 1001) + self.assertEqual(stat.S_IMODE(os.stat(path + '.1').st_mode), 0o640) + self.assertFalse(os.path.exists(path + '.4')) + + def test_missing_file_is_not_fatal(self): + self.assertFalse(rot.rotate_if_needed('/nonexistent/api-access.log', 10, 3)) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test-haproxy-stderr-relay.py b/scripts/test-haproxy-stderr-relay.py new file mode 100644 index 0000000..214c833 --- /dev/null +++ b/scripts/test-haproxy-stderr-relay.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build).""" +import importlib.util +import io +import os +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py')) +relay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(relay) + + +class Clock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +class RelayTests(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-') + self.path = os.path.join(self.dir, 'haproxy-stderr.log') + open(self.path, 'w').close() + self.out = io.StringIO() + self.clock = Clock() + + def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3): + return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock) + + def append(self, text): + with open(self.path, 'a') as fh: + fh.write(text) + + def test_history_is_not_replayed_and_new_lines_are_prefixed(self): + self.append('old line from a previous container\n') + r = self.make() + r.poll_once() + self.assertEqual(self.out.getvalue(), '') + self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n') + r.poll_once() + self.assertEqual(self.out.getvalue(), + 'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n') + + def test_partial_line_is_held_until_complete(self): + r = self.make() + r.poll_once() + self.append('Thread 1 is abo') + r.poll_once() + self.assertEqual(self.out.getvalue(), '') + self.append('ut to kill the process.\n') + r.poll_once() + self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue()) + + def test_control_characters_cannot_forge_lines(self): + r = self.make() + r.poll_once() + self.append('a\x1b[31mb\r\x00c\n') + r.poll_once() + self.assertEqual(self.out.getvalue().count('\n'), 1) + self.assertNotIn('\x1b', self.out.getvalue()) + + def test_rate_limit_passes_a_full_dump_then_summarises(self): + r = self.make(burst=200, per_min=60) + r.poll_once() + self.append(''.join('dump line %d\n' % i for i in range(150))) + r.poll_once() + self.assertEqual(self.out.getvalue().count('\n'), 150) + self.out.truncate(0), self.out.seek(0) + self.append(''.join('flap %d\n' % i for i in range(500))) + r.poll_once() + self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left + self.clock.t += 600 # bucket refills + self.out.truncate(0), self.out.seek(0) + self.append('after storm\n') + r.poll_once() + r.tick() + self.assertIn('[relay] suppressed ', self.out.getvalue()) + self.assertIn('after storm', self.out.getvalue()) + + def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self): + r = self.make(max_bytes=100, backups=2) + r.poll_once() + self.append('x' * 150 + '\n') + r.poll_once() + r.rotate_if_needed() + self.assertEqual(os.path.getsize(self.path), 0) + self.assertTrue(os.path.exists(self.path + '.1')) + self.append('after rotate\n') + r.poll_once() + self.assertIn('after rotate', self.out.getvalue()) + + def test_missing_file_is_not_an_error(self): + os.remove(self.path) + r = self.make() + r.poll_once() + self.append('late\n') + r.poll_once() + self.assertIn('late', self.out.getvalue()) + + + +# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic --------------- + +def reload_block(pid, n_proxies=100): + """What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config).""" + L = ['[NOTICE] (%d) : Reloading HAProxy' % pid, + '[NOTICE] (%d) : Initializing new worker (%d)' % (pid, pid + 100), + '[NOTICE] (%d) : haproxy version is 3.2.26-1~bpo13+1' % (pid + 100), + '[NOTICE] (%d) : path to executable is /usr/sbin/haproxy' % (pid + 100)] + for i in range(6): + L.append("[WARNING] (%d) : config : parsing [/etc/haproxy/haproxy.cfg:%d] : a 'http-request' rule " + "placed after a 'use_backend' rule will still be processed before." % (pid + 100, 800 + i)) + L.append("[WARNING] (%d) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid) + L.append('[NOTICE] (%d) : Loading success.' % pid) + for i in range(n_proxies): + L.append('[WARNING] (%d) : Proxy be_%d stopped (cumulated conns: FE: 0, BE: 12).' % (pid + 50, i)) + L.append('[WARNING] (%d) : Former worker (%d) exited with code 0 (Exit)' % (pid, pid + 50)) + return L + + +DUMP = [ + 'Thread 1 is about to kill the process (pid 6631).', + '* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0', + ' call trace(15):', + ' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0', + ' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' lock_hist: R:TASK_WQ U:TASK_WQ', + ' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + '', + "Hint: when reporting this bug to developers, please check if a core file was", + " produced, open it with 'gdb', issue 't a a bt full', check that the", + '[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)', + '[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.', + 'Please check that you are running an up to date and maintained version of haproxy and open a bug report.', + 'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/', + 'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html', + '[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM', + '[WARNING] (6328) : All workers exited. Exiting... (134)', +] +KEYS = ('about to kill', 'Thread 1 :', 'Thread 2 :', 'Thread 3 :', 'Thread 4 :', 'Current worker', + 'exited with code 134', 'exit-on-failure', 'All workers exited', 'call trace', 'ha_dump_backtrace', + 'bug in haproxy', 'Please check that you are running', 'produced, open it with') + + +def scenario(mod, burst=400, drained=True): + """4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts.""" + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', out, burst, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + lines = [] + for k in range(4): + lines += reload_block(100 * (k + 1)) + n_before = len(lines) + lines += DUMP + for l in lines: + if l.strip(): # poll_once drops blank lines before _emit + r._emit(l) + t[0] += 0.003 + r.tick() + return {'out': out.getvalue(), 'err': err.getvalue(), 'reload_lines': n_before, 'bucket': r.tokens, 'r': r} + + +class CapturedScenarioTests(unittest.TestCase): + def test_drained_bucket_then_panic_forwards_every_crash_line(self): + f = scenario(relay) + self.assertGreater(f['reload_lines'], 400) # would drain a 400-line bucket + e = f['err'] + for k in KEYS: + self.assertIn(k, e, k) + # the whole dump, line for line (blank lines are dropped by design) + got = [l[len(relay.PREFIX):] for l in e.split('\n') if l] + self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()]) + + def test_dump_is_forwarded_in_full_even_with_an_empty_bucket(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in ('benign %d' % i for i in range(10)): + r._emit(l) + self.assertEqual(out.getvalue(), '') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertEqual(err.getvalue().count('\n'), len([d for d in DUMP if d.strip()])) + self.assertEqual(r.suppressed, 10) + + def test_reload_noise_is_collapsed_not_drained(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for k in range(10): + for l in reload_block(100 * (k + 1)): + r._emit(l) + self.assertEqual(r.tokens, 400.0) # bucket untouched by 1000+ noise lines + self.assertEqual(r.suppressed, 0) + self.assertEqual(err.getvalue(), '') + summ = [l for l in out.getvalue().split('\n') if 'reload/start ok' in l] + self.assertEqual(len(summ), 10) # exactly one per reload + self.assertNotIn('Proxy be_', out.getvalue()) + self.assertNotIn('parsing [', out.getvalue()) + self.assertLess(out.getvalue().count('\n'), 15) + + def test_suppressed_lines_are_summarised_per_minute(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(105): + r._emit('flapping server state %d' % i) + self.assertNotIn('[relay] suppressed', out.getvalue()) + t[0] += 61 + r.tick() + self.assertRegex(out.getvalue(), r'\[relay\] suppressed 100 lines') + self.assertEqual(r.suppressed, 0) + self.assertEqual(err.getvalue(), '') + + def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr(self): + # docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "" is NOT parsed + f = scenario(relay) + self.assertIn('[relay] reload/start ok', f['out']) + self.assertNotIn('Thread', f['out']) + self.assertNotIn('Current worker', f['out']) + self.assertNotIn('[ALERT]', f['out']) + self.assertNotIn('reload/start ok', f['err']) + for text in (f['out'], f['err']): + for l in text.split('\n'): + self.assertFalse(l.startswith('<') or l.startswith(relay.PREFIX + '<'), l) + + def test_clean_worker_exit_is_not_crash_evidence(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit('[WARNING] (1) : Former worker (5) exited with code 0 (Exit)') + r._emit('[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)') + self.assertEqual(err.getvalue().count('\n'), 1) + self.assertIn('code 143', err.getvalue()) + + def test_crash_classes_never_limited(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)', + '[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM', + '[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)', + '[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy', + '[WARNING] (6328) : All workers exited. Exiting... (134)', + ' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + '* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0'] + for l in evidence: + r._emit(l) + self.assertEqual(err.getvalue().count('\n'), len(evidence)) + + FORGED = [ + 'GET /FATAL/SIGSEGV/segfault HTTP/1.1', + '[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout', + '[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN', + '[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)', + 'Host: Current worker (1) exited with code 134 (Aborted)', + 'Host: [ALERT] (1) : forged alert', + 'x-forwarded-for: Thread 1 is about to kill the process (pid 1).', + 'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1', + 'ha_panic watchdog FATAL SIGABRT', + 'backend ALERT_Current worker_exited with code 134', + '[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN', + ] + + def test_forged_keywords_stay_rate_limited(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in self.FORGED * 50: + r._emit(l) + self.assertEqual(err.getvalue(), '') + self.assertFalse(r.in_dump) + self.assertEqual(r.suppressed, len(self.FORGED) * 50) + self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget + # a forged header in the middle of normal traffic cannot open a block either + r._emit('proxy: Thread 1 is about to kill the process (pid 1).') + r._emit('free text that would be a dump line') + self.assertFalse(r.in_dump) + self.assertEqual(err.getvalue(), '') + + def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit("GET /Loading success HTTP/1.1") + self.assertEqual(r.routine, 0) + self.assertIn('Loading success', out.getvalue()) + + def test_flood_of_dump_headers_is_capped_by_the_window_budget(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(20000): + r._emit('Thread 1 is about to kill the process (pid %d).' % i) + r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0') + r._emit('[WARNING] (1) : All workers exited. Exiting... (134)') + n = err.getvalue().count('\n') + self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) + self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently + t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again + before = err.getvalue().count('\n') + r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)') + self.assertEqual(err.getvalue().count('\n'), before + 1) + + def test_bound_closed_block_cannot_rearm_during_cooldown(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 5): + r._emit('never closed %d' % i) + self.assertFalse(r.in_dump) + r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header + self.assertFalse(r.in_dump) + r._emit('call trace line after the second header') + self.assertNotIn('call trace line after', err.getvalue()) + t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again + r._emit('Thread 1 is about to kill the process (pid 3).') + self.assertTrue(r.in_dump) + + def test_hostile_and_very_long_lines(self): + r = self.make_relay() + r.poll_once() + self.append('A' * 500000 + '\n') + self.append('\x1b]0;evil\x07 [ALERT] forged\x00\r\n') + r.poll_once() + outs = self.out.getvalue().split('\n') + self.assertTrue(all(len(l) < relay.MAX_LINE + 200 for l in outs)) + self.assertNotIn('\x1b', self.out.getvalue()) + self.assertNotIn('\x00', self.out.getvalue()) + self.assertEqual(self.out.getvalue().count('\n'), 2) + + def test_unterminated_flood_is_bounded_in_memory(self): + r = self.make_relay() + r.poll_once() + for _ in range(3): + self.append('B' * 400000) # never a newline + r.poll_once() + self.assertLessEqual(len(r.pending), relay.MAX_PENDING) + + def test_unclosed_dump_block_cannot_disable_the_limiter(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 500): + r._emit('never closed %d' % i) + self.assertFalse(r.in_dump) + self.assertLessEqual(err.getvalue().count('\n'), relay.DUMP_MAX_LINES + 3) + self.assertGreaterEqual(r.suppressed, 400) + + def test_dump_block_is_closed_by_time_bound(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + t[0] += relay.DUMP_MAX_SECONDS + 1 + r._emit('later benign line') + self.assertFalse(r.in_dump) + self.assertNotIn('later benign', err.getvalue()) + + + # --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence -------- + TENANT_LINES = [ + "[ALERT] (10) : backend 'be_tenant1' has no server available!", + "[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.", + "[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.", + "[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.", + ] + + def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit(self.TENANT_LINES[0]) + t[0] += 0.001 + t[0] += 5.0 + for l in DUMP: + if l.strip(): + r._emit(l) + got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l] + self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()]) + self.assertIn('exited with code 134', err.getvalue()) + self.assertNotIn('exited with code 134', out.getvalue()) + self.assertNotIn('has no server available', err.getvalue()) + + def test_tenant_lines_are_their_own_rate_limited_stdout_class(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)]) + self.assertEqual(err.getvalue(), '') # never PRIORITY 3 + self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2) + self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped + self.assertEqual(r.tokens, 400.0) # general bucket untouched + self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched + self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET) + self.assertGreater(r.tenant_suppressed, 1000) + t[0] += relay.SUMMARY_EVERY + 1 + r.tick() + self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines') + self.assertEqual(r.tenant_suppressed, 0) + + def test_other_alerts_are_not_crash_evidence(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit('[ALERT] (12) : config : unable to load') + r._emit("[ALERT] (12) : backend 'x' has no server available!") + self.assertEqual(err.getvalue(), '') + self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr + + + def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop + r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i) + self.assertEqual(r.budget_left, 0) + before = err.getvalue().count('\n') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()])) + + def test_rotated_backup_is_mode_0640(self): + r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0) + os.chmod(self.path, 0o644) + self.append('x' * 100 + '\n') + r.rotate_if_needed() + self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640) + + # shared helpers for the file-driven tests + def setUp(self): + self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-') + self.path = os.path.join(self.dir, 'haproxy-stderr.log') + open(self.path, 'w').close() + self.out = io.StringIO() + + def make_relay(self): + return relay.Relay(self.path, self.out, 1000, 120, 10 ** 6, 3, clock=lambda: 0.0) + + def append(self, text): + with open(self.path, 'a') as fh: + fh.write(text) + + +# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------ + +def _scenario_ok(mod): + f = scenario(mod) + got = [l[len(mod.PREFIX):] for l in f['err'].split('\n') if l] + crash_ok = all(k in f['err'] for k in KEYS) and got == [d.rstrip() for d in DUMP if d.strip()] + collapse_ok = f['out'].count('reload/start ok') == 4 and 'Proxy be_' not in f['out'] and f['bucket'] > 300 + prio_ok = 'Thread' not in f['out'] and 'reload/start ok' not in f['err'] + return crash_ok, collapse_ok, prio_ok + + +class MutantTests(unittest.TestCase): + SRC = os.path.join(HERE, 'haproxy-stderr-relay.py') + + def load_mutant(self, old, new): + with open(self.SRC) as fh: + src = fh.read() + self.assertIn(old, src, 'mutation target vanished: update the test') + mutated = src.replace(old, new, 1) + self.assertNotEqual(mutated, src) + d = tempfile.mkdtemp(prefix='hap-relay-mutant-') # private copy, the real file is untouched + path = os.path.join(d, 'mutant.py') + with open(path, 'w') as fh: + fh.write(mutated) + sp = importlib.util.spec_from_file_location('mutant', path) + m = importlib.util.module_from_spec(sp) + sp.loader.exec_module(m) + return m + + def test_unmutated_copy_passes_everything(self): + self.assertEqual(_scenario_ok(relay), (True, True, True)) + + def test_mutant_limiter_applied_to_crash_lines(self): + m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:') + crash_ok, _, _ = _scenario_ok(m) + self.assertFalse(crash_ok) + + def test_mutant_dump_block_not_tracked(self): + m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:') + crash_ok, _, _ = _scenario_ok(m) + self.assertFalse(crash_ok) + + def test_mutant_collapse_disabled(self): + m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:') + _, collapse_ok, _ = _scenario_ok(m) + self.assertFalse(collapse_ok) + + def test_mutant_crash_lines_on_stdout(self): + m = self.load_mutant('self._write(self.err, line)', 'self._write(self.out, line)') + _, _, prio_ok = _scenario_ok(m) + self.assertFalse(prio_ok) + + def test_mutant_dump_bound_removed(self): + m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ') + err = io.StringIO() + r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(m.DUMP_MAX_LINES if False else 800): + r._emit('never closed %d' % i) + self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block... + + def _forged_run(self, mod): + err = io.StringIO() + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in CapturedScenarioTests.FORGED * 20: + r._emit(l) + return err.getvalue() + + def test_forged_run_clean_on_real_module(self): + self.assertEqual(self._forged_run(relay), '') + + def test_mutant_anchors_removed(self): + m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', + "if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):") + self.assertNotEqual(self._forged_run(m), '') + + def _flood(self, mod): + err, t = io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(5000): + r._emit('Thread 1 is about to kill the process (pid %d).' % i) + r._emit('[WARNING] (1) : All workers exited. Exiting... (134)') + return err.getvalue().count('\n') + + def test_flood_capped_on_real_module(self): + self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) + + def test_mutant_budget_removed(self): + m = self.load_mutant('if self._budget(now, kind):', 'if True:') + self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) + + def _rearm(self, mod): + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO()) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 5): + r._emit('x %d' % i) + r._emit('Thread 1 is about to kill the process (pid 2).') + return r.in_dump + + def test_mutant_cooldown_removed(self): + self.assertFalse(self._rearm(relay)) + m = self.load_mutant(' and now >= self.cooldown_until:', ':') + self.assertTrue(self._rearm(m)) + + def test_mutant_summary_removed(self): + m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines") + out, t = io.StringIO(), [0.0] + r = m.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=io.StringIO()) + for i in range(105): + r._emit('x %d' % i) + t[0] += 61 + r.tick() + self.assertNotIn('[relay] suppressed', out.getvalue()) + + + def _tenant_then_panic_ok(self, mod): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!") + t[0] += 0.001 + t[0] += 5.0 + for l in DUMP: + if l.strip(): + r._emit(l) + got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l] + return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue() + + def test_tenant_then_panic_clean_on_real_module(self): + self.assertTrue(self._tenant_then_panic_ok(relay)) + + def test_mutant_wide_alert_exemption_restored(self): + m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker") + self.assertFalse(self._tenant_then_panic_ok(m)) + + def test_mutant_tenant_class_removed(self): + m = self.load_mutant('if TENANT_RE.match(line):', 'if False:') + self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m)) + + def _tenant_stdout_ok(self, mod): + out, err = io.StringIO(), io.StringIO() + r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + for i in range(1200): + r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout") + return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2 + + def test_mutant_shared_budget(self): + m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'") + out, err = io.StringIO(), io.StringIO() + r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for i in range(m.CRASH_BUDGET + 500): + r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i) + before = err.getvalue().count('\n') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()])) + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test-haproxy-supervisor.py b/scripts/test-haproxy-supervisor.py new file mode 100644 index 0000000..777c8df --- /dev/null +++ b/scripts/test-haproxy-supervisor.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH).""" +import os +import stat +import subprocess +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh') + +PIDOF = """#!/bin/sh +# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds +[ -f "$STATE/started" ] || exit 1 +start=$(cat "$STATE/started") +now=$(date +%s.%N) +awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}' +""" +PYTHON = """#!/bin/sh +# stand-in for ensure_haproxy.py: stamp the start, record it +date +%s.%N > "$STATE/started" +date +%s.%N >> "$STATE/starts" +exit ${ENSURE_RC:-0} +""" + + +class SupervisorTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp(prefix='hap-sup-') + self.state = os.path.join(self.d, 'state') + os.mkdir(self.state) + for name, body in (('pidof', PIDOF), ('python', PYTHON)): + p = os.path.join(self.d, name) + with open(p, 'w') as fh: + fh.write(body) + os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR) + + def run_sup(self, seconds, **env): + e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state) + e.setdefault('LIFE', '0') + e.update(env) + try: + subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + return 'exited' + except subprocess.TimeoutExpired: + return 'running' # still looping when the clock ran out = alive + + def starts(self): + try: + with open(os.path.join(self.state, 'starts')) as fh: + return [float(x) for x in fh.read().split()] + except OSError: + return [] + + def test_non_integer_interval_does_not_kill_the_supervisor(self): + self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running') + self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying + + def test_garbage_interval_and_max_fall_back(self): + self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running') + self.assertGreaterEqual(len(self.starts()), 1) + + def test_crash_loop_backs_off_2_5_10_capped(self): + # haproxy dies immediately after every successful start (LIFE=0) -> quick deaths + self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3', + HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8', + HAPROXY_SUPERVISOR_MAX_BACKOFF='3') + s = self.starts() + gaps = [round(b - a, 1) for a, b in zip(s, s[1:])] + self.assertGreaterEqual(len(s), 4, gaps) + # gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4 + self.assertLess(gaps[0], gaps[1] + 0.3, gaps) + self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps) + self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps) + self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps) + + def test_stable_haproxy_resets_the_backoff(self): + # alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth + self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3', + HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8', + HAPROXY_SUPERVISOR_MAX_BACKOFF='3') + s = self.starts() + gaps = [b - a for a, b in zip(s, s[1:])] + self.assertGreaterEqual(len(s), 3, gaps) + self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top + + def test_running_haproxy_is_left_alone(self): + with open(os.path.join(self.state, 'started'), 'w') as fh: + fh.write(str(time.time())) + self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running') + self.assertEqual(self.starts(), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test-verify-published-tags.py b/scripts/test-verify-published-tags.py new file mode 100755 index 0000000..e189e3d --- /dev/null +++ b/scripts/test-verify-published-tags.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Offline tests for scripts/verify-published-tags.sh (final publish gate). + +CI run 5381 failed because buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'` +and prints the whole inspect text, while buildx 0.38 prints a bare digest. A stub +`imagetools` emulates BOTH output styles (plus `--raw`), so no registry or Docker +is needed. Held invariants: + - the 0.28-style (full text) and 0.38-style (bare digest) outputs both verify + - a reported digest that disagrees with the sha256 of the raw manifest fails + - a missing tag, a platform mismatch, or different digests across registries / + tags fail the job + - build-push.yaml's verify step uses the script, and release/* + rc-* tags + resolve to ONLY -rc and (never latest) + +Running: python3 scripts/test-verify-published-tags.py (needs jq) +""" +import hashlib +import json +import os +import re +import stat +import subprocess +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) +SCRIPT = os.path.join(HERE, 'verify-published-tags.sh') +WORKFLOW = os.path.join(ROOT, '.gitea', 'workflows', 'build-push.yaml') + +STUB = r'''#!/usr/bin/env python3 +import hashlib, json, os, sys +a = sys.argv[1:] +mode = os.environ['STUB_MODE'] # v028 | v038 +table = json.load(open(os.environ['STUB_TABLE'])) # ref -> {"raw": str, "lie": optional digest} +assert a[0] == 'inspect' +ref = [x for x in a[1:] if not x.startswith('-') and not x.startswith('{{')][0] +ent = table.get(ref) +if ent is None: + sys.stderr.write('ERROR: %s: not found\n' % ref); sys.exit(1) +raw = ent['raw'].encode() + (b'\n' if ent.get('nl') else b'') +digest = ent.get('lie') or 'sha256:' + hashlib.sha256(ent['raw'].encode()).hexdigest() +if '--raw' in a: + sys.stdout.buffer.write(raw); sys.exit(0) +if '--format' in a: + if mode == 'v038': + print(digest) + else: # 0.28: --format ignored, whole text printed + print('Name: ' + ref); print('MediaType: application/vnd.oci.image.index.v1+json') + print('Digest: ' + digest) + sys.exit(0) +print('Name: ' + ref) +print('MediaType: application/vnd.oci.image.index.v1+json') +print('Digest: ' + digest) +print(' ') +print('Manifests: ') +for m in json.loads(ent['raw'])['manifests']: + print(' Name: %s@%s' % (ref, m['digest'])) + print(' Digest: sha256:' + 'f' * 64) # indented: must never be mistaken for the index digest + print(' Platform: %s/%s' % (m['platform']['os'], m['platform']['architecture'])) +''' + + +def index(archs, salt=''): + ms = [{'mediaType': 'application/vnd.oci.image.manifest.v1+json', + 'digest': 'sha256:' + hashlib.sha256((x + salt).encode()).hexdigest(), 'size': 1, + 'platform': {'architecture': x, 'os': 'linux'}} for x in archs] + ms.append({'mediaType': 'application/vnd.oci.image.manifest.v1+json', + 'digest': 'sha256:' + '7' * 64, 'size': 1, + 'platform': {'architecture': 'unknown', 'os': 'unknown'}, + 'annotations': {'vnd.docker.reference.type': 'attestation-manifest'}}) + return json.dumps({'schemaVersion': 2, 'mediaType': 'application/vnd.oci.image.index.v1+json', + 'manifests': ms}, separators=(',', ':')) + + +REPOS = ['reg.one/a/img', 'reg.two/b/img'] +TAGS = ['1.2-rc', 'abc123'] + + +class VerifyTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.stub = os.path.join(self.tmp.name, 'stub') + open(self.stub, 'w').write(STUB) + os.chmod(self.stub, os.stat(self.stub).st_mode | stat.S_IEXEC) + self.table = os.path.join(self.tmp.name, 'table.json') + + def tearDown(self): + self.tmp.cleanup() + + def table_for(self, raw, **over): + t = {'%s:%s' % (r, g): {'raw': raw} for r in REPOS for g in TAGS} + for k, v in over.items(): + ref = k.replace('__', ':').replace('_', '/', 0) + t[ref] = v + return t + + def run_script(self, table, mode, platforms='linux/amd64,linux/arm64', tags=TAGS): + json.dump(table, open(self.table, 'w')) + args = [SCRIPT, '--platforms', platforms] + for r in REPOS: + args += ['--repo', r] + for t in tags: + args += ['--tag', t] + env = dict(os.environ, IMAGETOOLS=self.stub, STUB_MODE=mode, STUB_TABLE=self.table) + return subprocess.run(args, env=env, capture_output=True, text=True, timeout=60) + + def test_buildx_028_full_text_output_verifies(self): + r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v028') + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn('verified:', r.stdout) + self.assertNotIn('bad digest', r.stdout) + + def test_buildx_038_bare_digest_output_verifies(self): + r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v038') + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_trailing_newline_on_raw_output_tolerated(self): + raw = index(['amd64', 'arm64']) + t = {'%s:%s' % (r, g): {'raw': raw, 'nl': True} for r in REPOS for g in TAGS} + self.assertEqual(self.run_script(t, 'v028').returncode, 0) + + def test_amd64_only_publish_matches_platforms(self): + raw = index(['amd64']) + self.assertEqual(self.run_script(self.table_for(raw), 'v028', 'linux/amd64').returncode, 0) + r = self.run_script(self.table_for(raw), 'v028', 'linux/amd64,linux/arm64') + self.assertEqual(r.returncode, 1) + self.assertIn('platform mismatch', r.stdout) + + def test_missing_tag_fails(self): + t = self.table_for(index(['amd64', 'arm64'])) + del t['reg.two/b/img:abc123'] + r = self.run_script(t, 'v038') + self.assertEqual(r.returncode, 1) + self.assertIn('MISSING', r.stdout) + + def test_different_digest_across_registries_fails(self): + t = self.table_for(index(['amd64', 'arm64'])) + t['reg.two/b/img:1.2-rc'] = {'raw': index(['amd64', 'arm64'], salt='other')} + for mode in ('v028', 'v038'): + r = self.run_script(t, mode) + self.assertEqual(r.returncode, 1, mode) + self.assertIn('digest mismatch', r.stdout) + + def test_reported_digest_disagreeing_with_raw_manifest_fails(self): + t = self.table_for(index(['amd64', 'arm64'])) + t['reg.one/a/img:1.2-rc']['lie'] = 'sha256:' + '0' * 64 + for mode in ('v028', 'v038'): + r = self.run_script(t, mode) + self.assertEqual(r.returncode, 1, mode) + self.assertIn('cross-check FAILED', r.stdout) + + def test_garbage_manifest_fails(self): + t = self.table_for('not json') + self.assertEqual(self.run_script(t, 'v038').returncode, 1) + + +class WorkflowTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + import yaml + cls.src = open(WORKFLOW).read() + cls.doc = yaml.safe_load(cls.src) + cls.steps = cls.doc['jobs']['Build-and-Push']['steps'] + + def step(self, name): + return [s for s in self.steps if s.get('name') == name][0] + + def test_verify_step_uses_script_not_format_digest(self): + run = self.step('Verify published tags in every registry')['run'] + self.assertIn('scripts/verify-published-tags.sh', run) + self.assertNotIn('{{.Manifest.Digest}}', run) + + def test_run_blocks_are_valid_bash(self): + for s in self.steps: + if 'run' in s: + p = subprocess.run(['bash', '-n'], input=s['run'], text=True, capture_output=True) + self.assertEqual(p.returncode, 0, '%s: %s' % (s.get('name'), p.stderr)) + + def test_release_and_rc_never_publish_latest(self): + run = self.step('Resolve tags')['run'] + m = re.search(r'refs/heads/release/\*\|refs/tags/rc-\*\)\s*\n\s*tags="([^"]*)"', run) + self.assertIsNotNone(m) + self.assertEqual(m.group(1), '${VERSION}-rc ${SHA}') + main = re.search(r'refs/heads/main\)\s*\n\s*tags="([^"]*)"', run) + self.assertIsNotNone(main) + self.assertEqual(main.group(1), 'latest ${VERSION} ${SHA}') + # latest appears in no other tag assignment + self.assertEqual(len(re.findall(r'tags="[^"]*latest', run)), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/verify-published-tags.sh b/scripts/verify-published-tags.sh new file mode 100755 index 0000000..3ee6905 --- /dev/null +++ b/scripts/verify-published-tags.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Final publish gate: every : must resolve, carry exactly the +# platforms the assemble step reported, and have ONE index digest across all +# tags and all registries. Any miss or mismatch exits 1. +# +# verify-published-tags.sh --platforms linux/amd64,linux/arm64 \ +# --repo [--repo ...] --tag [--tag ...] +# +# The index digest is computed in a buildx-VERSION-INDEPENDENT way. CI run 5381 +# showed buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'` and prints the +# whole inspect text, while 0.38 prints a bare digest. So the digest comes from: +# 1. sha256 of the raw manifest bytes (`inspect --raw`) - always, authoritative +# (a content digest IS the sha256 of the manifest bytes); +# 2. the strictly anchored `Digest:` line of the default text output, if present; +# 3. the `--format '{{.Manifest.Digest}}'` output, ONLY if it is a bare digest +# (0.28's full-text output is ignored here, not an error). +# Every source that is available must agree, else the ref fails. +# Platforms are read from the raw JSON (also version independent). +# +# Env: IMAGETOOLS overrides the command (default "docker buildx imagetools"); +# the CI step sets it to "$BUILDX imagetools"; the test suite uses a stub. +set -uo pipefail + +IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools} +platforms=""; repos=(); tags=() +while [ $# -gt 0 ]; do + case "$1" in + --platforms) platforms=${2-}; shift 2 ;; + --repo) repos+=("${2-}"); shift 2 ;; + --tag) tags+=("${2-}"); shift 2 ;; + *) echo "verify-published-tags: unknown argument: $1" >&2; exit 2 ;; + esac +done +[ -n "$platforms" ] && [ ${#repos[@]} -gt 0 ] && [ ${#tags[@]} -gt 0 ] \ + || { echo "::error::verify-published-tags: --platforms, --repo and --tag are required"; exit 2; } +command -v jq >/dev/null || { echo "::error::verify-published-tags: jq is required"; exit 2; } + +want=$(printf '%s' "$platforms" | tr ',' '\n' | sort | tr '\n' ' ') +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +digest_re='^sha256:[0-9a-f]{64}$' +ref_digest=""; ref_name=""; fail=0 + +for r in "${repos[@]}"; do + for t in "${tags[@]}"; do + ref="${r}:${t}" + raw="$work/raw" + if ! timeout 90 $IMAGETOOLS inspect --raw "$ref" >"$raw" 2>"$work/err" || [ ! -s "$raw" ]; then + echo "::error::MISSING or unreadable: ${ref}: $(head -c 300 "$work/err")"; fail=1; continue + fi + # Raw bytes as served, and with trailing newlines stripped (some buildx + # versions append one when printing); the content digest is one of the two. + h1="sha256:$(sha256sum <"$raw" | cut -d' ' -f1)" + h2="sha256:$(printf '%s' "$(cat "$raw")" | sha256sum | cut -d' ' -f1)" + + # Source 2: strictly anchored top-level "Digest:" line of the text output. + textd="" + if timeout 90 $IMAGETOOLS inspect "$ref" >"$work/text" 2>/dev/null; then + mapfile -t lines < <(grep -E '^Digest:[[:space:]]+sha256:[0-9a-f]{64}[[:space:]]*$' "$work/text" | sed -E 's/^Digest:[[:space:]]+//; s/[[:space:]]+$//') + [ ${#lines[@]} -eq 1 ] && textd=${lines[0]} + fi + # Source 3: bare --format output, only when it really is a bare digest. + fmtd="" + if out=$(timeout 90 $IMAGETOOLS inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null); then + out=$(printf '%s' "$out" | tr -d '[:space:]') + [[ "$out" =~ $digest_re ]] && fmtd=$out + fi + + d="" + for cand in "$textd" "$fmtd"; do + [ -n "$cand" ] || continue + if [ "$cand" != "$h1" ] && [ "$cand" != "$h2" ]; then + echo "::error::digest cross-check FAILED for ${ref}: reported ${cand}, raw manifest hashes to ${h1}"; fail=1; d="!"; break + fi + d=$cand + done + [ "$d" = "!" ] && continue + [ -n "$d" ] || d=$h2 + [[ "$d" =~ $digest_re ]] || { echo "::error::bad digest for ${ref}: ${d}"; fail=1; continue; } + if [ -z "$textd$fmtd" ]; then + echo "::warning::no reported digest for ${ref}; using sha256 of the raw manifest" + fi + + if ! got=$(jq -r '.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"' "$raw" 2>/dev/null | sort | tr '\n' ' '); then + echo "::error::unparseable manifest for ${ref}"; fail=1; continue + fi + if [ "$got" != "$want" ]; then + echo "::error::platform mismatch for ${ref}: have [${got}] want [${want}]"; fail=1 + fi + echo "ok ${ref} ${d} [${got}]" + if [ -z "$ref_digest" ]; then ref_digest="$d"; ref_name="$ref" + elif [ "$d" != "$ref_digest" ]; then + echo "::error::digest mismatch: ${ref}=${d} vs ${ref_name}=${ref_digest}"; fail=1 + fi + done +done +[ "$fail" = 0 ] || { echo "::error::tag verification FAILED"; exit 1; } +echo "verified: ${tags[*]} in ${#repos[@]} registries, digest ${ref_digest}, platforms ${platforms}"