HAProxy 3.2 LTS edge image (default for all new deployments) #20
No files matched your search
@@ -1,9 +1,15 @@
|
||||
name: HAProxy Manager Build and Push
|
||||
run-name: ${{ gitea.actor }} pushed a change to main
|
||||
run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }}
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
# Release candidates: a push to release/* builds the SAME image but publishes
|
||||
# ONLY :<VERSION>-rc and :<sha>, never :latest and never the plain :<VERSION>
|
||||
# (see the "Resolve tags" step). Main keeps its behaviour.
|
||||
- 'release/**'
|
||||
tags:
|
||||
- 'rc-*'
|
||||
# Manual publish (e.g. after a [skip ci] merge, or to re-publish after a
|
||||
# failed arm64 leg).
|
||||
workflow_dispatch:
|
||||
@@ -77,6 +83,30 @@ jobs:
|
||||
id: ver
|
||||
run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Publishing strategy. main: :latest + :<version> + :<sha> (unchanged).
|
||||
# release/* branch or rc-* tag: :<version>-rc + :<sha> ONLY. A release candidate
|
||||
# is deployed per host by explicit tag (durable per-host pin in WHP); :latest is
|
||||
# moved to it by a later main publish only after the staged rollout succeeded.
|
||||
# Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused
|
||||
# rather than guessed at.
|
||||
- name: Resolve tags
|
||||
id: tags
|
||||
env:
|
||||
REF: ${{ gitea.ref }}
|
||||
VERSION: ${{ steps.ver.outputs.version }}
|
||||
SHA: ${{ gitea.sha }}
|
||||
run: |
|
||||
case "$REF" in
|
||||
refs/heads/main)
|
||||
tags="latest ${VERSION} ${SHA}" ;;
|
||||
refs/heads/release/*|refs/tags/rc-*)
|
||||
tags="${VERSION}-rc ${SHA}" ;;
|
||||
*)
|
||||
echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;;
|
||||
esac
|
||||
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
|
||||
echo "publishing tags: ${tags}"
|
||||
|
||||
# amd64: required. Pushed untagged by digest to both registries (same
|
||||
# digest in each); tags are only created by the assemble step below.
|
||||
- name: Build amd64 (push by digest)
|
||||
@@ -103,19 +133,79 @@ jobs:
|
||||
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
|
||||
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
# Three tags per registry: :latest (moving), :<version> (human-readable
|
||||
# release), :<sha> (immutable, guaranteed-unique rollback target).
|
||||
- name: Assemble tag indexes
|
||||
# imagetools needs a NEW buildx. Same pinned, sha256-verified binary as
|
||||
# mirror-base-image.yaml (kept identical on purpose). The assemble step
|
||||
# previously ran on whatever buildx the (possibly ephemeral) runner had
|
||||
# (0.13.1 on the runner that hung in run 5375, 0.15.1 on the trusted one).
|
||||
- name: Install pinned buildx
|
||||
env:
|
||||
BUILDX_VERSION: v0.28.0
|
||||
BUILDX_SHA256: 696bc104bac3bb708eff1af3f8bbc09fda0fd88f5757c1f9b404a35117889224
|
||||
run: |
|
||||
set -euo pipefail
|
||||
dir="${RUNNER_TEMP:-/tmp}/pinned-buildx"; mkdir -p "$dir"
|
||||
curl -fsSL --retry 3 --max-time 120 -o "$dir/docker-buildx" \
|
||||
"https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
|
||||
echo "${BUILDX_SHA256} $dir/docker-buildx" | sha256sum -c -
|
||||
chmod +x "$dir/docker-buildx"
|
||||
"$dir/docker-buildx" version
|
||||
echo "BUILDX=$dir/docker-buildx" >> "$GITHUB_ENV"
|
||||
|
||||
# main: three tags per registry: :latest (moving), :<version> (human-readable
|
||||
# release), :<sha> (immutable, guaranteed-unique rollback target).
|
||||
# release/* or rc-*: :<version>-rc and :<sha> only (see "Resolve tags").
|
||||
# Hardened after run 5375 (assemble hung after pushing, TTY progress, old
|
||||
# buildx): plain progress, pinned buildx, hard 10 minute step timeout.
|
||||
- name: Assemble tag indexes
|
||||
id: assemble
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
BUILDKIT_PROGRESS: plain
|
||||
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
|
||||
ARM64_DIGEST: ${{ steps.arm64.outputs.digest }}
|
||||
ARM64_OUTCOME: ${{ steps.arm64.outcome }}
|
||||
VERSION: ${{ steps.ver.outputs.version }}
|
||||
SHA: ${{ gitea.sha }}
|
||||
TAGS: ${{ steps.tags.outputs.tags }}
|
||||
run: |
|
||||
scripts/assemble-multiarch-index.sh \
|
||||
set -euo pipefail
|
||||
tag_args=()
|
||||
for t in $TAGS; do tag_args+=(--tag "$t"); done
|
||||
# Use the pinned buildx for create AND the script's platform inspects.
|
||||
export IMAGETOOLS="${BUILDX} imagetools"
|
||||
out=$(scripts/assemble-multiarch-index.sh \
|
||||
--amd64 "$AMD64_DIGEST" \
|
||||
--arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \
|
||||
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
|
||||
--repo ghcr.io/shadowdao/haproxy-manager-base \
|
||||
--tag latest --tag "$VERSION" --tag "$SHA"
|
||||
"${tag_args[@]}" | tee /dev/stderr)
|
||||
platforms=$(printf '%s\n' "$out" | sed -n 's/^platforms=//p' | tail -n1)
|
||||
[ -n "$platforms" ] || { echo "::error::assemble script reported no platforms line"; exit 1; }
|
||||
if [ "$platforms" = "linux/amd64" ]; then
|
||||
echo "::warning::SINGLE-ARCH publish (amd64 only) on purpose: arm64 outcome was '${ARM64_OUTCOME}'. Tags carry no arm64 manifest."
|
||||
else
|
||||
echo "multi-arch publish: ${platforms}"
|
||||
fi
|
||||
echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Final gate: every tag in every registry must resolve, carry the platforms
|
||||
# the assemble step reported, and have ONE digest across all tags and both
|
||||
# registries (they are built from the same sources). Any miss fails the job
|
||||
# instead of trusting that the create calls "looked" fine.
|
||||
- name: Verify published tags in every registry
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
BUILDKIT_PROGRESS: plain
|
||||
TAGS: ${{ steps.tags.outputs.tags }}
|
||||
PLATFORMS: ${{ steps.assemble.outputs.platforms }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Version-independent digest logic lives in the script (tested by
|
||||
# scripts/test-verify-published-tags.py); buildx 0.28 ignores --format.
|
||||
export IMAGETOOLS="${BUILDX} imagetools"
|
||||
tag_args=()
|
||||
for t in $TAGS; do tag_args+=(--tag "$t"); done
|
||||
scripts/verify-published-tags.sh \
|
||||
--platforms "$PLATFORMS" \
|
||||
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
|
||||
--repo ghcr.io/shadowdao/haproxy-manager-base \
|
||||
"${tag_args[@]}"
|
||||
@@ -14,7 +14,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
- **WAF body inspection**: `python3 scripts/test-waf-body-inspection.py` - offline (runs in the image build); pins `option http-buffer-request`, the `X-WHP-WAF-Body` complete/partial stamp (strip -> stamp -> SPOE -> strip), `tune.bufsize` against the 100 ms SPOE fail-open budget and the 65535-byte SPOP frame cap, and Coraza's body limits in `coraza-spoa/config.yaml`. Run it after touching the Coraza block, `tune.bufsize`, or the SPOE engine.
|
||||
- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired/<stamp>/` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/<primary>` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`.
|
||||
- **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch").
|
||||
- **Publish verify gate**: `python3 scripts/test-verify-published-tags.py` - offline (needs `jq`; NOT run in the image build); `scripts/verify-published-tags.sh` is the last build-push.yaml step: every tag in every registry must resolve, carry the assembled platforms, and share ONE index digest. The digest is the sha256 of `imagetools inspect --raw` cross-checked against the text `Digest:` line / bare `--format` output (buildx 0.28 ignores `--format` and prints full text; CI run 5381).
|
||||
- **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash".
|
||||
- **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines.
|
||||
- **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`.
|
||||
- **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md
|
||||
|
||||
@@ -178,4 +180,5 @@ re-reads it on reload, so a failed runtime command must degrade to
|
||||
- **The :8000 API refuses peers other than 127.0.0.1/::1 and the container's default gateway** (`HAPROXY_MGR_API_ALLOWED_PEERS`, default `127.0.0.1,::1,gateway`; ports via `HAPROXY_MGR_PEER_FILTER_PORTS`, default `8000`). haproxy-manager shares `client-net` with every tenant container; host callers arrive from the gateway via docker-proxy. Only effective with the WHP `-p 127.0.0.1:8000:8000` publish (a 0.0.0.0 publish lets a tenant hairpin through the host and arrive from the gateway). A sidecar that must call :8000 directly needs its IP added
|
||||
- gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat
|
||||
- Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/`
|
||||
- HAProxy is version 3.0.11
|
||||
- HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101)
|
||||
- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (dump blocks opened only by the anchored `Thread N is about to kill` header, `[ALERT] (PID) :`, `Current worker (N) exited with code N`; ANCHORED patterns, so keywords echoed from URLs/Host/server names never match) bypasses the bucket within a budget (1000 lines / 10 min, 60 s cooldown after a bound closes a block) and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `<N>` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay
|
||||
+58
-26
@@ -23,49 +23,78 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \
|
||||
org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# haproxy is PINNED to a reviewed UPSTREAM version (HAPROXY_UPSTREAM), not to one
|
||||
# Debian package revision. It was previously unpinned, so the binary could move
|
||||
# under us at Debian's timing - an upstream release that rejected our config would
|
||||
# have broken an unrelated commit's build, or worse, shipped an edge that refuses
|
||||
# to start (see the `haproxy -c` gate below for why that matters: a config HAProxy
|
||||
# rejects leaves the container Up with 80/443 unbound and /health still 200).
|
||||
# haproxy comes from the UPSTREAM-maintained apt repo https://haproxy.debian.net
|
||||
# (Vincent Bernat's builds, the Debian maintainer's own), suite
|
||||
# `trixie-backports-<series>`, NOT from Debian trixie's archive.
|
||||
#
|
||||
# History: this used to pin the exact Debian version (3.0.11-1+deb13u3). Debian
|
||||
# security then shipped 3.0.11-1+deb13u4 (same upstream 3.0.11, same arch set) and
|
||||
# the exact pin silently froze the image on the OLDER, unpatched revision, and
|
||||
# would hard-fail the build as soon as u3 left the archive. Debian security
|
||||
# revisions of the SAME upstream version are exactly what we want to take, so:
|
||||
# * an apt preference (Pin: version 3.0.11-*) makes apt install the NEWEST
|
||||
# Debian revision of that upstream version on any architecture;
|
||||
# * the build then FAILS unless the installed package AND `haproxy -v` both
|
||||
# report that upstream version, so a major/minor bump (3.0 -> 3.1, 3.0.12)
|
||||
# can never arrive by accident: that still needs this ARG changed and the
|
||||
# `haproxy -c` gate below to pass.
|
||||
# The pin and the gate compose: pinned = the version moves deliberately; the gate
|
||||
# answers immediately whether the new binary still accepts fleet config.
|
||||
# WHY (2026-11-16 forensics): Debian trixie ships 3.0.11 (+ security revisions).
|
||||
# That build aborts via HAProxy's own watchdog inside the DNS resolver
|
||||
# (resolv_process_responses / snr_resolution_cb, haproxy issue #3101) - whole
|
||||
# edge down 15-52 s per abort on sdbees/whp01. The fix ("dns: bind the
|
||||
# nameserver sockets to the initiating thread", "resolvers: make
|
||||
# process_resolvers() single-threaded") is in 3.0.13+, 3.1.11+ and 3.2.6+;
|
||||
# Debian trixie will not take it. See docs/go-port/haproxy-edge-upgrade-and-logging.md
|
||||
# in the WHP repo for the version evidence and the rollout plan.
|
||||
#
|
||||
# To move to another upstream version: change HAPROXY_UPSTREAM, rebuild, let the
|
||||
# gate verify. Check what Debian offers with: apt-cache madison haproxy
|
||||
ARG HAPROXY_UPSTREAM=3.0.11
|
||||
# Series: 3.2 (LTS, supported to 2030-Q2). Its trixie package is built with
|
||||
# USE_QUIC=1 against the system OpenSSL 3.5.x NATIVE QUIC API (depends on
|
||||
# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream
|
||||
# has no trixie packages and would need a source build.)
|
||||
#
|
||||
# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 ..
|
||||
# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any
|
||||
# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian
|
||||
# package version, HAPROXY_EXACT (default below), and the build:
|
||||
# * pins apt to it (`Pin: version <full Debian version>`; NOTE a bare
|
||||
# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1",
|
||||
# and with a non-matching pin apt falls back to Debian's own 3.0.11 package);
|
||||
# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v`
|
||||
# reports the series (so an unreachable repo, which would leave Debian's
|
||||
# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident);
|
||||
# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c`
|
||||
# gate below proves the binary accepts the fleet config.
|
||||
# Taking a new patch is a DELIBERATE edit of the default below (or
|
||||
# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never
|
||||
# passes it, so a rebuild is reproducible.
|
||||
# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat,
|
||||
# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with
|
||||
# signed-by, never fetched at build time. The fingerprint is asserted below.
|
||||
ARG HAPROXY_UPSTREAM=3.2
|
||||
ARG HAPROXY_EXACT=3.2.26-1~bpo13+1
|
||||
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
|
||||
# (CT130 pkg-cache, https only; CI sets it only after probing the cache with the
|
||||
# pinned CA). Unset = the upstream Debian repositories, exactly as before. The
|
||||
# apt sources are restored before this RUN ends (restore FAILS the build if any
|
||||
# reference to the cache is left), so no shipped layer carries the cache address.
|
||||
# NOTE: CT130 does not proxy haproxy.debian.net (see the design doc); that one
|
||||
# repo is always fetched directly.
|
||||
# Canonical script + docs: cloud-apache-container (scripts/pkg-cache-repos.sh,
|
||||
# docs/pkg-cache.md there); this repo carries byte-identical copies.
|
||||
ARG PKG_CACHE=
|
||||
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
|
||||
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
|
||||
COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/haproxy-archive-keyring.gpg
|
||||
RUN /usr/local/sbin/pkg-cache-repos.sh apt && \
|
||||
apt update -y && apt dist-upgrade -y && \
|
||||
printf 'Package: haproxy\nPin: version %s-*\nPin-Priority: 1001\n' "${HAPROXY_UPSTREAM}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
|
||||
apt install -y --no-install-recommends gnupg ca-certificates && \
|
||||
keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \
|
||||
[ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \
|
||||
[ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \
|
||||
apt purge -y gnupg && \
|
||||
echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \
|
||||
apt update -y && \
|
||||
pin="${HAPROXY_EXACT}" && \
|
||||
printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
|
||||
apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \
|
||||
rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \
|
||||
deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \
|
||||
case "${deb_ver}" in "${HAPROXY_UPSTREAM}"-*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not upstream ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
|
||||
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}-" && \
|
||||
echo "haproxy pin OK: upstream ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \
|
||||
[ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \
|
||||
case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
|
||||
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \
|
||||
haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \
|
||||
if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \
|
||||
mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \
|
||||
echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \
|
||||
/usr/local/sbin/pkg-cache-repos.sh restore && \
|
||||
apt clean && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /haproxy
|
||||
@@ -122,11 +151,14 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \
|
||||
&& python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \
|
||||
&& python3 /haproxy/scripts/test-default-splash.py \
|
||||
&& python3 /haproxy/scripts/test-access-log-receiver.py \
|
||||
&& python3 /haproxy/scripts/test-haproxy-stderr-relay.py \
|
||||
&& python3 /haproxy/scripts/test-haproxy-supervisor.py \
|
||||
&& python3 /haproxy/scripts/test-config-rollback.py \
|
||||
&& python3 /haproxy/scripts/test-cert-write-safety.py \
|
||||
&& python3 /haproxy/scripts/test-cert-scripts.py \
|
||||
&& python3 /haproxy/scripts/test-waf-body-inspection.py \
|
||||
&& python3 /haproxy/scripts/test-gunicorn-socket-timeouts.py \
|
||||
&& python3 /haproxy/scripts/test-gunicorn-access-logging.py \
|
||||
&& python3 /haproxy/scripts/validate-rendered-config.py
|
||||
# Create log directories
|
||||
RUN mkdir -p /var/log/haproxy && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
|
||||
|
||||
@@ -451,6 +451,8 @@ Covered by `scripts/test-default-splash.py` (template order/conditions, the
|
||||
page, list generation) and the `splash-off` scenario of
|
||||
`scripts/validate-rendered-config.py`.
|
||||
|
||||
> gunicorn access logs (:8080 none, :8000 bounded file, not stdout): see docs/logging.md.
|
||||
|
||||
## Access Log (in-container, since 2026.11.2)
|
||||
|
||||
HAProxy logs (`log /log.sock ... format rfc5424`, `hap_header.tpl`) to a UNIX
|
||||
@@ -492,6 +494,22 @@ Rollout: image bump + recreate with the volume and sysctl (WHP's
|
||||
are in the image, outside the `/etc/haproxy` named volume, so nothing can be
|
||||
shadowed; `haproxy.cfg` is regenerated on start with the new `log` line.
|
||||
|
||||
## HAProxy stderr (panic dumps, [ALERT]s) - since 2026.11.4
|
||||
|
||||
HAProxy is started as `haproxy -W -db` so the master keeps a real stderr (with
|
||||
the `daemon` keyword HAProxy points it at /dev/null, which discarded the
|
||||
`[ALERT]` lines, `Current worker (N) exited with code 134` and the whole
|
||||
watchdog/`ha_panic` thread dump). It is appended to
|
||||
`/var/log/haproxy/haproxy-stderr.log` (the `haproxy-logs` volume, so it
|
||||
survives a recreate) and `scripts/haproxy-stderr-relay.py` re-emits every line
|
||||
on the container's stderr prefixed `haproxy-stderr: ` (rate-limited, file
|
||||
rotated at 20 MiB x 3). `docker logs haproxy-manager` and whatever docker
|
||||
logging driver the host configured therefore carry the crash text; the
|
||||
per-request access log stays out of it. The in-container supervisor probes
|
||||
with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` seconds (default 2, was 15) and
|
||||
backs off to `HAPROXY_SUPERVISOR_MAX_BACKOFF` (30) only when a restart attempt
|
||||
fails. Measured (3.2.26, empty config): crash to listeners back = 1.2-2.0 s.
|
||||
|
||||
## Default Backend Configuration
|
||||
|
||||
With the default splash on (above), unrouted hosts no longer reach this page; it still serves the blocked-IP and suspended pages, and serves unrouted hosts again when `HAPROXY_DEFAULT_SPLASH=off`.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Container logging policy (what reaches `docker logs`)
|
||||
|
||||
`docker logs haproxy-manager` (and the docker logging driver: json-file today,
|
||||
journald -> Graylog under WHP PR #260) must carry only low-volume, high-value
|
||||
lines. Per-request logs stay out of it.
|
||||
|
||||
| Source | Destination | Volume |
|
||||
|---|---|---|
|
||||
| HAProxy edge access log (1.5M lines/day) | `/var/log/haproxy/access.log` via `haproxy-log-receiver.py` (20 MiB cap, gzip generations) | NOT in `docker logs` |
|
||||
| HAProxy master stderr (ALERT, crash dumps) | `haproxy-stderr.log` -> `haproxy-stderr-relay.py` -> stdout (benign, rate limited) / stderr (crash evidence) | rare |
|
||||
| Supervisor / relay / init lines | stdout / stderr | rare |
|
||||
| gunicorn :8080 default-page / blocked-IP / suspended-site | NO access log | none |
|
||||
| gunicorn :8000 management API access log | `/var/log/haproxy/api-access.log` (haproxy-logs volume, mode 0640), rotated by `scripts/gunicorn-access-rotate.py` (copy-truncate, 20 MiB x 3) | NOT in `docker logs` |
|
||||
| gunicorn error log (tracebacks, worker timeouts) | stderr (`--error-logfile -`) | rare |
|
||||
|
||||
## Why
|
||||
|
||||
Before this change both gunicorns ran with `--access-logfile -`, so each :8080
|
||||
hit and each :8000 API call was one docker stdout line. With the journald
|
||||
driver that is one journal entry per request: journald's default rate limit
|
||||
(10000 messages / 30 s per sending service) can then suppress the very
|
||||
crash-dump lines the relay emits, and Graylog would be flooded.
|
||||
|
||||
## Knobs
|
||||
|
||||
`HAPROXY_MGR_API_ACCESS_LOG` (path), `HAPROXY_API_ACCESS_LOG_MAX_BYTES`
|
||||
(20971520), `HAPROXY_API_ACCESS_LOG_BACKUPS` (3), `HAPROXY_API_ACCESS_LOG_POLL`
|
||||
(30 s; the live file can exceed the cap by at most one interval of API traffic).
|
||||
|
||||
## Not done
|
||||
|
||||
Failed-auth (401/403) lines are NOT mirrored to stdout; they are in the file
|
||||
(grep the status column). Add a rate-limited mirror later if wanted.
|
||||
|
||||
## Tests
|
||||
|
||||
`python3 scripts/test-gunicorn-access-logging.py` (runs in the image build):
|
||||
real gunicorn, a burst of default-page requests produces zero stdout lines,
|
||||
API requests land in the file not stdout, handler exceptions still reach
|
||||
stderr, and rotation is bounded.
|
||||
|
||||
Deploying this needs a NEW rc image built from `feat/haproxy-upgrade`
|
||||
(VERSION stays 2026.11.5 until then).
|
||||
+89
-16
@@ -766,6 +766,92 @@ def is_process_running(process_name):
|
||||
return True
|
||||
return False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launching the HAProxy master (2026-11 forensics change)
|
||||
#
|
||||
# haproxy.cfg carries `daemon`, and in daemon mode HAProxy re-points the
|
||||
# master's stdin/stdout/stderr at /dev/null after it has parsed the config
|
||||
# (MEASURED on 3.0.11: /proc/<master>/fd/2 -> /dev/null). That discarded every
|
||||
# [ALERT], the "exited with code 134" master line and the whole ha_panic()/
|
||||
# watchdog thread dump of the 2026-11 aborts: only the core file survived.
|
||||
#
|
||||
# `-db` overrides `daemon` (MEASURED: foreground master, still master-worker,
|
||||
# reload via the CLI keeps the same master pid), so the master keeps a real
|
||||
# stderr. We append it to HAPROXY_STDERR_LOG (haproxy-logs volume, survives a
|
||||
# container recreate) and scripts/haproxy-stderr-relay.py forwards it to the
|
||||
# container's stderr (-> docker logging driver -> host journald/Graylog).
|
||||
#
|
||||
# The master is started as `sh -c '... &'` so it is re-parented to PID 1 exactly
|
||||
# as the old daemon fork did (PID 1 reaps it; a Popen child would sit as a
|
||||
# zombie in the API worker and is_process_running('haproxy') would keep
|
||||
# reporting it alive).
|
||||
# ---------------------------------------------------------------------------
|
||||
HAPROXY_STDERR_LOG = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
||||
try:
|
||||
HAPROXY_START_WAIT = float(os.environ.get('HAPROXY_START_WAIT', '10'))
|
||||
except ValueError:
|
||||
HAPROXY_START_WAIT = 10.0
|
||||
|
||||
|
||||
def _stderr_log_path():
|
||||
"""HAPROXY_STDERR_LOG if its directory is writable, else a /tmp fallback
|
||||
(a missing volume must never stop HAProxy from starting)."""
|
||||
path = HAPROXY_STDERR_LOG
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
# 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable)
|
||||
os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640))
|
||||
return path
|
||||
except OSError:
|
||||
return '/tmp/haproxy-stderr.log'
|
||||
|
||||
|
||||
def spawn_haproxy_master():
|
||||
"""Start `haproxy -W -db` detached, stderr appended to the stderr log.
|
||||
|
||||
Returns a subprocess.CompletedProcess (returncode 0, .stderr = the new log
|
||||
text) once HAProxy has loaded its configuration; raises
|
||||
subprocess.CalledProcessError (.stderr = what HAProxy printed) if it exits
|
||||
or never comes up within HAPROXY_START_WAIT seconds. Same contract the old
|
||||
blocking `subprocess.run([... '-W' ...], check=True)` call sites relied on.
|
||||
"""
|
||||
cmd = ['haproxy', '-W', '-db', '-S', '/tmp/haproxy-cli,level,admin',
|
||||
'-f', HAPROXY_CONFIG_PATH]
|
||||
log_path = _stderr_log_path()
|
||||
try:
|
||||
offset = os.path.getsize(log_path)
|
||||
except OSError:
|
||||
offset = 0
|
||||
subprocess.run(
|
||||
['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; '
|
||||
'"$@" </dev/null >>"$log" 2>&1 &', 'sh', log_path] + cmd,
|
||||
check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
|
||||
def new_text():
|
||||
try:
|
||||
with open(log_path, 'rb') as fh:
|
||||
fh.seek(offset)
|
||||
return fh.read().decode('utf-8', 'replace')
|
||||
except OSError:
|
||||
return ''
|
||||
|
||||
deadline = time.monotonic() + HAPROXY_START_WAIT
|
||||
started = time.monotonic()
|
||||
while time.monotonic() < deadline:
|
||||
text = new_text()
|
||||
if 'Loading success.' in text:
|
||||
return subprocess.CompletedProcess(cmd, 0, '', text)
|
||||
if time.monotonic() - started > 1.0 and not is_process_running('haproxy'):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
text = new_text()
|
||||
if is_process_running('haproxy') and '[ALERT]' not in text:
|
||||
# Slow to log (big config) but alive: treat as started.
|
||||
return subprocess.CompletedProcess(cmd, 0, '', text)
|
||||
raise subprocess.CalledProcessError(1, cmd, output='', stderr=text[-4000:])
|
||||
|
||||
|
||||
# Initialize template engine
|
||||
template_loader = jinja2.FileSystemLoader(TEMPLATE_DIR)
|
||||
template_env = jinja2.Environment(loader=template_loader)
|
||||
@@ -871,12 +957,7 @@ def reload_haproxy():
|
||||
return jsonify({'status': 'success'}), 200
|
||||
else:
|
||||
# Start HAProxy if it's not running
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
if result.returncode == 0:
|
||||
print("HAProxy started successfully")
|
||||
log_operation('start_haproxy', True)
|
||||
@@ -3778,10 +3859,7 @@ def reload_haproxy_safely(backup_status=None):
|
||||
else:
|
||||
# HAProxy not running, start it
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True, capture_output=True, text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
logger.info("HAProxy started successfully")
|
||||
# Now - and only now - is this config known good.
|
||||
promote_current_config_to_backup()
|
||||
@@ -4114,12 +4192,7 @@ def start_haproxy():
|
||||
return
|
||||
|
||||
# Configuration is valid, start HAProxy
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
if result.returncode == 0:
|
||||
logger.info("HAProxy started successfully")
|
||||
log_operation('start_haproxy', True, 'HAProxy started successfully')
|
||||
|
||||
Binary file not shown.
@@ -34,6 +34,18 @@ def main():
|
||||
haproxy_manager.logger.warning(
|
||||
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
|
||||
)
|
||||
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
|
||||
# before it vanished (the relay forwards the same text; this ties it to the
|
||||
# supervisor event even if the relay is behind).
|
||||
try:
|
||||
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
|
||||
fh.seek(0, 2)
|
||||
fh.seek(max(0, fh.tell() - 1500))
|
||||
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
|
||||
for line in tail:
|
||||
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
|
||||
except OSError:
|
||||
pass
|
||||
# start_haproxy() validates the config (and regenerates it if invalid)
|
||||
# before launching, and swallows its own errors, so it will not raise here.
|
||||
haproxy_manager.start_haproxy()
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bound the management-API (:8000) gunicorn access log file.
|
||||
|
||||
Why
|
||||
---
|
||||
Gunicorn's access log used to go to stdout (`--access-logfile -`), i.e. every
|
||||
:8080 default-page / blocked-IP / suspended-site hit and every :8000 API call
|
||||
became a `docker logs` line. With the journald docker log driver that is one
|
||||
journal entry per request (journald's default rate limit, 10000 msgs / 30 s per
|
||||
sending service, can then SUPPRESS the crash-dump lines the relay emits) and it
|
||||
floods Graylog. Now:
|
||||
* :8080 (default page) has NO access log at all;
|
||||
* :8000 (API) writes its access log to API_ACCESS_LOG (default
|
||||
/var/log/haproxy/api-access.log, in the haproxy-logs volume), NOT stdout;
|
||||
* gunicorn ERROR logging (tracebacks, worker timeouts) stays on stderr.
|
||||
This process keeps that file bounded: copy-truncate rotation at
|
||||
HAPROXY_API_ACCESS_LOG_MAX_BYTES (default 20 MiB), HAPROXY_API_ACCESS_LOG_BACKUPS
|
||||
generations (default 3), rotated copies mode 0640 (same policy as
|
||||
haproxy-stderr.log). Truncating in place is safe because gunicorn opens the
|
||||
file in append mode. Several gunicorn processes share the file, so renaming it
|
||||
is not an option.
|
||||
|
||||
Bound: size is checked every HAPROXY_API_ACCESS_LOG_POLL seconds (default 30),
|
||||
so the live file can exceed the limit only by what the API writes in one
|
||||
interval (a few hundred bytes per call).
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _env_num(name, default, cast=int):
|
||||
try:
|
||||
v = cast(os.environ.get(name, default))
|
||||
return v if v > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def rotate_if_needed(path, max_bytes, backups):
|
||||
"""Copy-truncate rotate. Returns True if it rotated."""
|
||||
try:
|
||||
if os.path.getsize(path) <= max_bytes:
|
||||
return False
|
||||
for i in range(backups, 1, -1):
|
||||
src = '%s.%d' % (path, i - 1)
|
||||
if os.path.exists(src):
|
||||
os.replace(src, '%s.%d' % (path, i))
|
||||
shutil.copyfile(path, path + '.1')
|
||||
os.chmod(path + '.1', 0o640)
|
||||
with open(path, 'r+b') as fh:
|
||||
fh.truncate(0)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except OSError as exc:
|
||||
sys.stderr.write('gunicorn-access-rotate: rotate failed: %s\n' % exc)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
path = os.environ.get('HAPROXY_MGR_API_ACCESS_LOG', '/var/log/haproxy/api-access.log')
|
||||
max_bytes = _env_num('HAPROXY_API_ACCESS_LOG_MAX_BYTES', 20 * 1024 * 1024)
|
||||
backups = _env_num('HAPROXY_API_ACCESS_LOG_BACKUPS', 3)
|
||||
poll = _env_num('HAPROXY_API_ACCESS_LOG_POLL', 30.0, float)
|
||||
while True:
|
||||
try:
|
||||
rotate_if_needed(path, max_bytes, backups)
|
||||
except Exception as exc: # never die; the supervisor loop restarts us anyway
|
||||
sys.stderr.write('gunicorn-access-rotate: %r\n' % exc)
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+363
@@ -0,0 +1,363 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
|
||||
|
||||
Why
|
||||
---
|
||||
HAProxy's master stderr carries what the access log never will: `[ALERT]`
|
||||
lines, "Current worker (N) exited with code 134", and the complete
|
||||
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
|
||||
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
|
||||
haproxy-logs volume, so it survives a container recreate). This process tails the
|
||||
file and re-emits each line on ITS stderr, prefixed, which is the container's
|
||||
stderr: `docker logs`, and through the docker logging driver the host journal
|
||||
and Graylog.
|
||||
|
||||
Properties
|
||||
----------
|
||||
* Rare by nature: HAProxy only writes here on start/reload/server state
|
||||
changes/crashes. The per-request access log is NOT here (it goes to the
|
||||
receiver socket), so this does not feed the WHP AI monitor a firehose.
|
||||
* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE
|
||||
("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is
|
||||
about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/
|
||||
server name never matches) and every line of a dump block (from a panic/watchdog header to
|
||||
the master's "All workers exited" line, bounded by
|
||||
DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block
|
||||
can open; blocks open only on the anchored "Thread N is about to kill" header)
|
||||
are forwarded in full even when the bucket is empty, within a total budget of
|
||||
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
|
||||
and are counted in the suppressed summary). They go to STDERR.
|
||||
* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!",
|
||||
"Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints
|
||||
the first at [ALERT]: every tenant container stop/crash/deploy makes them.
|
||||
Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own
|
||||
"[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI
|
||||
monitor through `docker logs haproxy-manager`; rate-limiting them is intended.
|
||||
Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend.
|
||||
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
|
||||
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
|
||||
"Initializing new worker", "Loading success", ...) never touches the bucket:
|
||||
it is counted and collapsed into ONE "[relay] reload/start ok: N routine
|
||||
lines collapsed" line per reload (plus one on the minute tick for the late
|
||||
"Proxy stopped" lines). The full text stays in the stderr file.
|
||||
* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST
|
||||
lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute
|
||||
"[relay] suppressed N lines" summary.
|
||||
* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and
|
||||
stderr -> PRIORITY 3 (err) and does NOT parse "<N>" prefixes. So benign
|
||||
lines (routine, rate-limited remainder, relay summaries) are written to
|
||||
STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority.
|
||||
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
|
||||
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
|
||||
Truncating is safe because HAProxy's fd is O_APPEND.
|
||||
* Never blocks or affects HAProxy: it only reads a file.
|
||||
|
||||
Environment
|
||||
-----------
|
||||
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
|
||||
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
|
||||
HAPROXY_STDERR_RELAY_POLL (0.5 s)
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
PREFIX = 'haproxy-stderr: '
|
||||
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
|
||||
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
|
||||
DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this
|
||||
DUMP_MAX_SECONDS = 30.0
|
||||
SUMMARY_EVERY = 60.0
|
||||
|
||||
# Every pattern is ANCHORED to HAProxy's own message structure (captured from
|
||||
# real 3.2 crashes): a keyword inside a request path / Host header / server
|
||||
# name that HAProxy echoes into a warning can never match, because the line
|
||||
# prefix "[ALERT] (PID) :" is written by HAProxy itself.
|
||||
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
|
||||
CRASH_RE = re.compile(
|
||||
r'^(?:'
|
||||
r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
|
||||
r'|Thread \d+ is about to kill the process'
|
||||
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
|
||||
r')')
|
||||
# Tenant-driven state lines. WHP gives every tenant server `check`, so every
|
||||
# tenant container stop/crash/OOM/deploy and every host-wide container restart
|
||||
# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash
|
||||
# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so
|
||||
# they can neither reach priority 3 nor spend the general bucket / crash budgets.
|
||||
TENANT_RE = re.compile(
|
||||
r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:'
|
||||
r"backend '[^']*' has no server available!"
|
||||
r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)'
|
||||
r'|Health check for (?:backup )?server \S+ '
|
||||
r')')
|
||||
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
|
||||
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
|
||||
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
|
||||
# A new start/reload cycle also closes a dump block.
|
||||
CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)')
|
||||
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
|
||||
ROUTINE_RE = re.compile(
|
||||
r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:'
|
||||
r'Proxy \S+ stopped \(cumulated conns'
|
||||
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
|
||||
r'|Former worker \(\d+\) exited with code 0\b'
|
||||
r'|Reloading HAProxy|Initializing new worker|Loading success'
|
||||
r'|path to executable is|haproxy version is'
|
||||
r"|config: Can't load stats-file|'option forwardfor' ignored)")
|
||||
LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success')
|
||||
# Total exemption budget: even real-looking evidence cannot flood without bound.
|
||||
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
|
||||
CRASH_WINDOW = 600.0 # seconds
|
||||
DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that
|
||||
# ordinary evidence lines (worker exits, alerts) can never spend
|
||||
TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute)
|
||||
TENANT_PER_MIN = 30
|
||||
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
|
||||
|
||||
|
||||
def _env_num(name, default, cast=int):
|
||||
try:
|
||||
v = cast(os.environ.get(name, default))
|
||||
return v if v > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None):
|
||||
self.path = path
|
||||
self.out = out # benign lines (docker journald: PRIORITY 6)
|
||||
self.err = err if err is not None else out # crash lines (PRIORITY 3)
|
||||
self.burst, self.per_min = burst, per_min
|
||||
self.max_bytes, self.backups = max_bytes, backups
|
||||
self.clock = clock
|
||||
self.tokens = float(burst)
|
||||
self.last = clock()
|
||||
self.suppressed = 0
|
||||
self.routine = 0 # collapsed routine lines since the last summary
|
||||
self.summary_at = clock()
|
||||
self.in_dump = False
|
||||
self.dump_lines = 0
|
||||
self.dump_since = 0.0
|
||||
self.cooldown_until = 0.0
|
||||
self.budget_left = CRASH_BUDGET
|
||||
self.window_start = clock()
|
||||
self.dump_budget_left = DUMP_BUDGET
|
||||
self.dump_window_start = clock()
|
||||
self.tenant_tokens = float(TENANT_BURST)
|
||||
self.tenant_last = clock()
|
||||
self.tenant_suppressed = 0
|
||||
self.offset = None # None = first pass: start at EOF (do not replay history)
|
||||
self.pending = ''
|
||||
self._last_stream = None
|
||||
|
||||
def _take(self):
|
||||
now = self.clock()
|
||||
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
|
||||
self.last = now
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _take_tenant(self):
|
||||
now = self.clock()
|
||||
self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0)
|
||||
self.tenant_last = now
|
||||
if self.tenant_tokens >= 1:
|
||||
self.tenant_tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _write(self, stream, text):
|
||||
# keep the two streams in emission order: flush the other one first
|
||||
if self._last_stream is not None and self._last_stream is not stream:
|
||||
self._last_stream.flush()
|
||||
stream.write(PREFIX + text + '\n')
|
||||
self._last_stream = stream
|
||||
|
||||
def _flush_all(self):
|
||||
for st in {id(self.out): self.out, id(self.err): self.err}.values():
|
||||
try:
|
||||
st.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _summaries(self, force=False):
|
||||
"""Per-minute summary of what the limiter / collapse withheld."""
|
||||
now = self.clock()
|
||||
if not force and now - self.summary_at < SUMMARY_EVERY:
|
||||
return
|
||||
self.summary_at = now
|
||||
if self.routine:
|
||||
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
|
||||
% self.routine)
|
||||
self.routine = 0
|
||||
if self.tenant_suppressed:
|
||||
self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines '
|
||||
'(rate limit; full text in the stderr file)' % self.tenant_suppressed)
|
||||
self.tenant_suppressed = 0
|
||||
if self.suppressed:
|
||||
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
|
||||
% self.suppressed)
|
||||
self.suppressed = 0
|
||||
|
||||
def _budget(self, now, kind='crash'):
|
||||
"""Take one unit of a budget; False once the window's budget is spent.
|
||||
|
||||
kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget)."""
|
||||
if kind == 'dump':
|
||||
if now - self.dump_window_start >= CRASH_WINDOW:
|
||||
self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET
|
||||
if self.dump_budget_left > 0:
|
||||
self.dump_budget_left -= 1
|
||||
return True
|
||||
return False
|
||||
if now - self.window_start >= CRASH_WINDOW:
|
||||
self.window_start, self.budget_left = now, CRASH_BUDGET
|
||||
if self.budget_left > 0:
|
||||
self.budget_left -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _end_dump(self, now, why=None):
|
||||
self.in_dump = False
|
||||
if why: # closed by a bound, not by the master's end line: no re-arming for a while
|
||||
self.cooldown_until = now + DUMP_COOLDOWN
|
||||
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
|
||||
|
||||
def _emit(self, line):
|
||||
now = self.clock()
|
||||
if self.in_dump:
|
||||
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
|
||||
self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
|
||||
elif CYCLE_RE.match(line):
|
||||
self._end_dump(now)
|
||||
if self.in_dump or CRASH_RE.match(line):
|
||||
is_dump = self.in_dump or bool(DUMP_START_RE.match(line))
|
||||
kind = 'dump' if is_dump else 'crash'
|
||||
if self._budget(now, kind):
|
||||
# crash evidence: bypasses the bucket, the collapse and the summary
|
||||
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
|
||||
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
|
||||
if self.in_dump:
|
||||
self.dump_lines += 1
|
||||
self._write(self.err, line)
|
||||
if self.in_dump and DUMP_END_RE.match(line):
|
||||
self._end_dump(now)
|
||||
return
|
||||
if self.in_dump: # budget spent mid-block: the normal limiter takes over
|
||||
self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW))
|
||||
if ROUTINE_RE.match(line):
|
||||
self.routine += 1
|
||||
if LOADED_RE.match(line): # one summary per start/reload
|
||||
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
|
||||
% self.routine)
|
||||
self.routine = 0
|
||||
self._summaries()
|
||||
return
|
||||
if TENANT_RE.match(line):
|
||||
if self._take_tenant():
|
||||
self._write(self.out, line)
|
||||
else:
|
||||
self.tenant_suppressed += 1
|
||||
self._summaries()
|
||||
return
|
||||
if self._take():
|
||||
self._write(self.out, line)
|
||||
else:
|
||||
self.suppressed += 1
|
||||
self._summaries()
|
||||
|
||||
@staticmethod
|
||||
def _clean(line):
|
||||
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
|
||||
if len(line) > MAX_LINE:
|
||||
line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line)
|
||||
return line
|
||||
|
||||
def rotate_if_needed(self):
|
||||
try:
|
||||
if os.path.getsize(self.path) <= self.max_bytes:
|
||||
return
|
||||
for i in range(self.backups, 1, -1):
|
||||
src = '%s.%d' % (self.path, i - 1)
|
||||
if os.path.exists(src):
|
||||
os.replace(src, '%s.%d' % (self.path, i))
|
||||
shutil.copyfile(self.path, self.path + '.1')
|
||||
os.chmod(self.path + '.1', 0o640)
|
||||
with open(self.path, 'r+b') as fh:
|
||||
fh.truncate(0)
|
||||
self.offset = 0
|
||||
except OSError as exc:
|
||||
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
|
||||
|
||||
def poll_once(self):
|
||||
try:
|
||||
size = os.path.getsize(self.path)
|
||||
except OSError:
|
||||
if self.offset is None:
|
||||
self.offset = 0 # file not created yet: whatever appears is new
|
||||
return
|
||||
if self.offset is None:
|
||||
self.offset = size
|
||||
return
|
||||
if size < self.offset: # truncated by someone else
|
||||
self.offset = 0
|
||||
if size == self.offset:
|
||||
return
|
||||
with open(self.path, 'rb') as fh:
|
||||
fh.seek(self.offset)
|
||||
chunk = fh.read(1 << 20)
|
||||
self.offset += len(chunk)
|
||||
text = self.pending + chunk.decode('utf-8', 'replace')
|
||||
lines = text.split('\n')
|
||||
self.pending = lines.pop() # incomplete trailing line stays buffered
|
||||
if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line
|
||||
lines.append(self.pending)
|
||||
self.pending = ''
|
||||
for line in lines:
|
||||
line = self._clean(line)
|
||||
if line:
|
||||
self._emit(line)
|
||||
self._summaries()
|
||||
self._flush_all()
|
||||
|
||||
def tick(self):
|
||||
"""Idle-time housekeeping: emit the per-minute summaries even when no line arrives."""
|
||||
self._summaries()
|
||||
self._flush_all()
|
||||
|
||||
|
||||
def main():
|
||||
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
||||
relay = Relay(path, sys.stdout,
|
||||
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
|
||||
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
|
||||
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
|
||||
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
|
||||
err=sys.stderr)
|
||||
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
|
||||
sys.stdout.write('haproxy-stderr-relay: following %s\n' % path)
|
||||
sys.stdout.flush()
|
||||
ticks = 0
|
||||
while True:
|
||||
try:
|
||||
relay.poll_once()
|
||||
relay.tick()
|
||||
ticks += 1
|
||||
if ticks % 120 == 0:
|
||||
relay.rotate_if_needed()
|
||||
except Exception as exc: # never die: the supervisor loop restarts us anyway
|
||||
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# In-container haproxy supervisor (started in the background by start-up.sh).
|
||||
#
|
||||
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
|
||||
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
|
||||
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||||
# policy never fires, and haproxy is down until the external host watchdog
|
||||
# full-restarts the whole container minutes later (dropping every connection).
|
||||
# This loop revives haproxy in place within one interval.
|
||||
#
|
||||
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
|
||||
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
|
||||
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
|
||||
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
|
||||
# most of it detection latency + start; the default is now 2 s (was 15).
|
||||
#
|
||||
# Two backoffs:
|
||||
# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its
|
||||
# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
|
||||
# re-rendering/re-validating the same broken config every 2 s;
|
||||
# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH
|
||||
# seconds (default 10) of that start (a crash loop): extra wait before the
|
||||
# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default
|
||||
# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives
|
||||
# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death
|
||||
# after a long run is still revived within one interval.
|
||||
#
|
||||
# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5,
|
||||
# "abc") falls back to the default instead of killing this subshell under set -e.
|
||||
set -eo pipefail
|
||||
|
||||
_posint() { # _posint VALUE DEFAULT
|
||||
case "$1" in
|
||||
''|*[!0-9]*) echo "$2" ;;
|
||||
*) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;;
|
||||
esac
|
||||
}
|
||||
|
||||
INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)"
|
||||
MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)"
|
||||
QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)"
|
||||
STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)"
|
||||
STEPS=""
|
||||
for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do
|
||||
v="$(_posint "$s" 0)"
|
||||
[ "$v" -ge 1 ] && STEPS="${STEPS} ${v}"
|
||||
done
|
||||
[ -n "${STEPS}" ] || STEPS=" 2 5 10 30"
|
||||
|
||||
# step N (1-based, past the end = last), capped at MAX_BACKOFF
|
||||
_step() {
|
||||
local n="$1" i=0 v=0
|
||||
for s in ${STEPS}; do
|
||||
i=$((i + 1)); v="$s"
|
||||
[ "${i}" -ge "${n}" ] && break
|
||||
done
|
||||
[ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}"
|
||||
echo "${v}"
|
||||
}
|
||||
|
||||
delay="${INTERVAL}"
|
||||
last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet
|
||||
quick=0 # consecutive deaths within QUICK_DEATH of a start
|
||||
while true; do
|
||||
sleep "${delay}"
|
||||
if pidof -q haproxy; then
|
||||
delay="${INTERVAL}"
|
||||
if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then
|
||||
quick=0; last_start=-1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "${last_start}" -ge 0 ]; then
|
||||
up=$((SECONDS - last_start))
|
||||
if [ "${up}" -lt "${QUICK_DEATH}" ]; then
|
||||
quick=$((quick + 1))
|
||||
wait_s="$(_step "${quick}")"
|
||||
echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2
|
||||
sleep "${wait_s}"
|
||||
pidof -q haproxy && continue
|
||||
elif [ "${up}" -ge "${STABLE}" ]; then
|
||||
quick=0
|
||||
fi
|
||||
fi
|
||||
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
|
||||
delay="${INTERVAL}"
|
||||
last_start="${SECONDS}"
|
||||
else
|
||||
delay=$(( delay * 2 ))
|
||||
[ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}"
|
||||
fi
|
||||
done
|
||||
+42
-17
@@ -75,24 +75,49 @@ mkdir -p /var/log/haproxy /var/lib/haproxy
|
||||
|
||||
cron &
|
||||
|
||||
# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and
|
||||
# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs,
|
||||
# "exited with code 134", the watchdog/ha_panic thread dump) is appended to
|
||||
# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's
|
||||
# stderr so `docker logs` / the docker logging driver / Graylog get it. Started
|
||||
# BEFORE init.py so the first start's lines are followed. Restarted within 2 s
|
||||
# if it ever dies; it only reads a file, so it can never affect HAProxy.
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/haproxy-stderr-relay.py || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
# gunicorn access logs (incident-prevention for the journald log driver).
|
||||
# gunicorn used to log every request to stdout (`--access-logfile -`), so every
|
||||
# :8080 default-page / blocked-IP hit and every :8000 API call became a docker
|
||||
# log line -> journal entry -> Graylog line, and could trip journald's rate
|
||||
# limit and suppress the relay's crash-dump lines. Now:
|
||||
# :8080 no access log at all;
|
||||
# :8000 access log to a size-bounded FILE in the haproxy-logs volume (mode
|
||||
# 0640), rotated by scripts/gunicorn-access-rotate.py (copy-truncate,
|
||||
# 20 MiB x 3). Set HAPROXY_MGR_API_ACCESS_LOG to relocate it.
|
||||
# Gunicorn ERROR logging (--error-logfile -) stays on stderr -> `docker logs`.
|
||||
API_ACCESS_LOG="${HAPROXY_MGR_API_ACCESS_LOG:-/var/log/haproxy/api-access.log}"
|
||||
export HAPROXY_MGR_API_ACCESS_LOG="${API_ACCESS_LOG}"
|
||||
mkdir -p "$(dirname "${API_ACCESS_LOG}")"
|
||||
[ -e "${API_ACCESS_LOG}" ] || : > "${API_ACCESS_LOG}"
|
||||
chmod 0640 "${API_ACCESS_LOG}" 2>/dev/null || true
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/gunicorn-access-rotate.py || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
# Phase 1: container init
|
||||
python /haproxy/scripts/init.py
|
||||
|
||||
# Phase 1.5: in-container haproxy supervisor.
|
||||
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
|
||||
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
|
||||
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||||
# policy never fires, and haproxy is down until the external host watchdog
|
||||
# full-restarts the whole container minutes later (dropping every connection).
|
||||
# This loop revives haproxy in place within one interval. ensure_haproxy.py is
|
||||
# idempotent — a cheap no-op whenever haproxy is already running.
|
||||
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}"
|
||||
(
|
||||
while true; do
|
||||
sleep "${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
python /haproxy/scripts/ensure_haproxy.py 2>&1 || true
|
||||
done
|
||||
) &
|
||||
# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh:
|
||||
# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with
|
||||
# backoff on failed restarts and on crash loops; knobs validated as integers).
|
||||
/haproxy/scripts/haproxy-supervisor.sh &
|
||||
|
||||
# Phase 2: WSGI servers
|
||||
# Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT
|
||||
@@ -132,7 +157,7 @@ gunicorn \
|
||||
--workers 1 --threads 4 --worker-class "${WORKER_CLASS}" \
|
||||
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
|
||||
--timeout 30 \
|
||||
--access-logfile - --error-logfile - --log-level info \
|
||||
--error-logfile - --log-level info \
|
||||
--pythonpath /haproxy \
|
||||
'haproxy_manager:default_app' &
|
||||
|
||||
@@ -144,6 +169,6 @@ exec gunicorn \
|
||||
--workers "${API_WORKERS}" --threads 4 --worker-class "${WORKER_CLASS}" \
|
||||
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
|
||||
--timeout "${API_TIMEOUT}" \
|
||||
--access-logfile - --error-logfile - --log-level info \
|
||||
--access-logfile "${API_ACCESS_LOG}" --error-logfile - --log-level info \
|
||||
--pythonpath /haproxy \
|
||||
'haproxy_manager:app'
|
||||
@@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here.
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN}
|
||||
# reject any unusable file in $TEST_CERTS_DIR
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
@@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# real `haproxy -W -db` prints this once the config is loaded
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
@@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here (default is the
|
||||
# haproxy-logs volume path, not writable unprivileged).
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
|
||||
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
# A real `haproxy -W -db` prints this once the config is loaded;
|
||||
# spawn_haproxy_master() waits for it.
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests: gunicorn access logging must not flood the container's stdout.
|
||||
|
||||
Context: with the journald docker log driver every stdout line is a journal
|
||||
entry (default rate limit 10000 msgs/30 s per service could suppress crash-dump
|
||||
lines) and, via Graylog, a stored message. start-up.sh therefore gives :8080 no
|
||||
access log and sends :8000's to a bounded FILE; gunicorn errors stay on stderr.
|
||||
|
||||
Pins (static, from start-up.sh):
|
||||
* no gunicorn invocation uses `--access-logfile -`;
|
||||
* the :8080 invocation has no --access-logfile at all;
|
||||
* the :8000 invocation logs access to ${API_ACCESS_LOG}, errors to `-`.
|
||||
Pins (live, REAL gunicorn, same flag shapes):
|
||||
* a burst of default-page requests (no access log) writes ZERO lines to
|
||||
stdout and none to stderr;
|
||||
* a burst of API requests writes ZERO stdout lines, the access file receives
|
||||
them, and a handler exception still reaches STDERR (error logging intact);
|
||||
* scripts/gunicorn-access-rotate.py keeps the file bounded (copy-truncate,
|
||||
backups capped, rotated copy mode 0640).
|
||||
Runs in the Docker build gate.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
try:
|
||||
import gunicorn # noqa: F401
|
||||
except ImportError:
|
||||
sys.stderr.write('SKIP: gunicorn not installed\n')
|
||||
raise SystemExit(77)
|
||||
|
||||
spec = importlib.util.spec_from_file_location('rot', os.path.join(HERE, 'gunicorn-access-rotate.py'))
|
||||
rot = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(rot)
|
||||
|
||||
APP = textwrap.dedent('''\
|
||||
def app(environ, start_response):
|
||||
if environ['PATH_INFO'] == '/boom':
|
||||
raise RuntimeError('handler-exploded')
|
||||
start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '2')])
|
||||
return [b'ok']
|
||||
''')
|
||||
|
||||
|
||||
def free_port():
|
||||
s = socket.socket()
|
||||
s.bind(('127.0.0.1', 0))
|
||||
p = s.getsockname()[1]
|
||||
s.close()
|
||||
return p
|
||||
|
||||
|
||||
def start_script():
|
||||
with open(os.path.join(HERE, 'start-up.sh')) as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
class StaticFlags(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.src = start_script()
|
||||
# Each invocation = from "gunicorn \" to its app target line.
|
||||
self.inv = re.findall(r'^(?:exec )?gunicorn \\\n(?:.*\\\n)*?.*haproxy_manager:\w+.*$', self.src, re.M)
|
||||
|
||||
def test_two_invocations(self):
|
||||
self.assertEqual(len(self.inv), 2, self.inv)
|
||||
|
||||
def test_no_stdout_access_log(self):
|
||||
for i in self.inv:
|
||||
self.assertNotRegex(i, r'--access-logfile\s+-(\s|$)')
|
||||
|
||||
def test_default_page_has_no_access_log(self):
|
||||
d = [i for i in self.inv if '0.0.0.0:8080' in i]
|
||||
self.assertEqual(len(d), 1)
|
||||
self.assertNotIn('--access-logfile', d[0])
|
||||
self.assertIn('--error-logfile -', d[0])
|
||||
|
||||
def test_api_access_log_is_a_file(self):
|
||||
a = [i for i in self.inv if '0.0.0.0:8000' in i]
|
||||
self.assertEqual(len(a), 1)
|
||||
self.assertIn('--access-logfile "${API_ACCESS_LOG}"', a[0])
|
||||
self.assertIn('--error-logfile -', a[0])
|
||||
|
||||
def test_rotator_started(self):
|
||||
self.assertIn('gunicorn-access-rotate.py', self.src)
|
||||
|
||||
|
||||
class Live(unittest.TestCase):
|
||||
N_DEFAULT = 300
|
||||
N_API = 100
|
||||
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='gunicorn-access-')
|
||||
with open(os.path.join(self.dir, 'tapp.py'), 'w') as fh:
|
||||
fh.write(APP)
|
||||
|
||||
def run_gunicorn(self, extra, n, boom=False):
|
||||
port = free_port()
|
||||
out = open(os.path.join(self.dir, 'out-%d' % port), 'w+')
|
||||
err = open(os.path.join(self.dir, 'err-%d' % port), 'w+')
|
||||
p = subprocess.Popen(
|
||||
[sys.executable, '-m', 'gunicorn', '--bind', '127.0.0.1:%d' % port,
|
||||
'--workers', '1', '--threads', '4', '--error-logfile', '-', '--log-level', 'info',
|
||||
'--pythonpath', self.dir] + extra + ['tapp:app'],
|
||||
stdout=out, stderr=err)
|
||||
try:
|
||||
for _ in range(100):
|
||||
try:
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=2).read()
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(0.2)
|
||||
for _ in range(n):
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/x' % port, timeout=5).read()
|
||||
if boom:
|
||||
try:
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/boom' % port, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
p.terminate()
|
||||
p.wait(10)
|
||||
out.seek(0)
|
||||
err.seek(0)
|
||||
o, e = out.read(), err.read()
|
||||
out.close()
|
||||
err.close()
|
||||
return o, e
|
||||
|
||||
def test_default_page_burst_is_silent(self):
|
||||
out, err = self.run_gunicorn([], self.N_DEFAULT)
|
||||
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
|
||||
self.assertNotIn('GET /x', err)
|
||||
|
||||
def test_api_burst_goes_to_file_not_stdout_and_errors_stay_on_stderr(self):
|
||||
path = os.path.join(self.dir, 'api-access.log')
|
||||
open(path, 'w').close()
|
||||
out, err = self.run_gunicorn(['--access-logfile', path], self.N_API, boom=True)
|
||||
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
|
||||
self.assertNotIn('GET /x', err)
|
||||
with open(path) as fh:
|
||||
lines = fh.read().splitlines()
|
||||
self.assertGreaterEqual(len([l for l in lines if 'GET /x' in l]), self.N_API)
|
||||
self.assertIn('handler-exploded', err) # error logging intact
|
||||
|
||||
|
||||
class Rotation(unittest.TestCase):
|
||||
def test_bounded_and_mode(self):
|
||||
d = tempfile.mkdtemp(prefix='gunicorn-rot-')
|
||||
path = os.path.join(d, 'api-access.log')
|
||||
for _ in range(12):
|
||||
with open(path, 'ab') as fh:
|
||||
fh.write(b'x' * 1000 + b'\n')
|
||||
rot.rotate_if_needed(path, 4000, 3)
|
||||
files = sorted(os.listdir(d))
|
||||
self.assertLessEqual(len(files), 4, files) # live + 3 backups
|
||||
self.assertLessEqual(os.path.getsize(path), 4000 + 1001)
|
||||
self.assertEqual(stat.S_IMODE(os.stat(path + '.1').st_mode), 0o640)
|
||||
self.assertFalse(os.path.exists(path + '.4'))
|
||||
|
||||
def test_missing_file_is_not_fatal(self):
|
||||
self.assertFalse(rot.rotate_if_needed('/nonexistent/api-access.log', 10, 3))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,614 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py'))
|
||||
relay = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(relay)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.t = 0.0
|
||||
|
||||
def __call__(self):
|
||||
return self.t
|
||||
|
||||
|
||||
class RelayTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
|
||||
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
|
||||
open(self.path, 'w').close()
|
||||
self.out = io.StringIO()
|
||||
self.clock = Clock()
|
||||
|
||||
def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3):
|
||||
return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock)
|
||||
|
||||
def append(self, text):
|
||||
with open(self.path, 'a') as fh:
|
||||
fh.write(text)
|
||||
|
||||
def test_history_is_not_replayed_and_new_lines_are_prefixed(self):
|
||||
self.append('old line from a previous container\n')
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(),
|
||||
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
|
||||
def test_partial_line_is_held_until_complete(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('Thread 1 is abo')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('ut to kill the process.\n')
|
||||
r.poll_once()
|
||||
self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue())
|
||||
|
||||
def test_control_characters_cannot_forge_lines(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('a\x1b[31mb\r\x00c\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 1)
|
||||
self.assertNotIn('\x1b', self.out.getvalue())
|
||||
|
||||
def test_rate_limit_passes_a_full_dump_then_summarises(self):
|
||||
r = self.make(burst=200, per_min=60)
|
||||
r.poll_once()
|
||||
self.append(''.join('dump line %d\n' % i for i in range(150)))
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 150)
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append(''.join('flap %d\n' % i for i in range(500)))
|
||||
r.poll_once()
|
||||
self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left
|
||||
self.clock.t += 600 # bucket refills
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append('after storm\n')
|
||||
r.poll_once()
|
||||
r.tick()
|
||||
self.assertIn('[relay] suppressed ', self.out.getvalue())
|
||||
self.assertIn('after storm', self.out.getvalue())
|
||||
|
||||
def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self):
|
||||
r = self.make(max_bytes=100, backups=2)
|
||||
r.poll_once()
|
||||
self.append('x' * 150 + '\n')
|
||||
r.poll_once()
|
||||
r.rotate_if_needed()
|
||||
self.assertEqual(os.path.getsize(self.path), 0)
|
||||
self.assertTrue(os.path.exists(self.path + '.1'))
|
||||
self.append('after rotate\n')
|
||||
r.poll_once()
|
||||
self.assertIn('after rotate', self.out.getvalue())
|
||||
|
||||
def test_missing_file_is_not_an_error(self):
|
||||
os.remove(self.path)
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('late\n')
|
||||
r.poll_once()
|
||||
self.assertIn('late', self.out.getvalue())
|
||||
|
||||
|
||||
|
||||
# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic ---------------
|
||||
|
||||
def reload_block(pid, n_proxies=100):
|
||||
"""What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config)."""
|
||||
L = ['[NOTICE] (%d) : Reloading HAProxy' % pid,
|
||||
'[NOTICE] (%d) : Initializing new worker (%d)' % (pid, pid + 100),
|
||||
'[NOTICE] (%d) : haproxy version is 3.2.26-1~bpo13+1' % (pid + 100),
|
||||
'[NOTICE] (%d) : path to executable is /usr/sbin/haproxy' % (pid + 100)]
|
||||
for i in range(6):
|
||||
L.append("[WARNING] (%d) : config : parsing [/etc/haproxy/haproxy.cfg:%d] : a 'http-request' rule "
|
||||
"placed after a 'use_backend' rule will still be processed before." % (pid + 100, 800 + i))
|
||||
L.append("[WARNING] (%d) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid)
|
||||
L.append('[NOTICE] (%d) : Loading success.' % pid)
|
||||
for i in range(n_proxies):
|
||||
L.append('[WARNING] (%d) : Proxy be_%d stopped (cumulated conns: FE: 0, BE: 12).' % (pid + 50, i))
|
||||
L.append('[WARNING] (%d) : Former worker (%d) exited with code 0 (Exit)' % (pid, pid + 50))
|
||||
return L
|
||||
|
||||
|
||||
DUMP = [
|
||||
'Thread 1 is about to kill the process (pid 6631).',
|
||||
'* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0',
|
||||
' call trace(15):',
|
||||
' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0',
|
||||
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' lock_hist: R:TASK_WQ U:TASK_WQ',
|
||||
' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
'',
|
||||
"Hint: when reporting this bug to developers, please check if a core file was",
|
||||
" produced, open it with 'gdb', issue 't a a bt full', check that the",
|
||||
'[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
|
||||
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.',
|
||||
'Please check that you are running an up to date and maintained version of haproxy and open a bug report.',
|
||||
'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/',
|
||||
'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html',
|
||||
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
|
||||
'[WARNING] (6328) : All workers exited. Exiting... (134)',
|
||||
]
|
||||
KEYS = ('about to kill', 'Thread 1 :', 'Thread 2 :', 'Thread 3 :', 'Thread 4 :', 'Current worker',
|
||||
'exited with code 134', 'exit-on-failure', 'All workers exited', 'call trace', 'ha_dump_backtrace',
|
||||
'bug in haproxy', 'Please check that you are running', 'produced, open it with')
|
||||
|
||||
|
||||
def scenario(mod, burst=400, drained=True):
|
||||
"""4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts."""
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', out, burst, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
lines = []
|
||||
for k in range(4):
|
||||
lines += reload_block(100 * (k + 1))
|
||||
n_before = len(lines)
|
||||
lines += DUMP
|
||||
for l in lines:
|
||||
if l.strip(): # poll_once drops blank lines before _emit
|
||||
r._emit(l)
|
||||
t[0] += 0.003
|
||||
r.tick()
|
||||
return {'out': out.getvalue(), 'err': err.getvalue(), 'reload_lines': n_before, 'bucket': r.tokens, 'r': r}
|
||||
|
||||
|
||||
class CapturedScenarioTests(unittest.TestCase):
|
||||
def test_drained_bucket_then_panic_forwards_every_crash_line(self):
|
||||
f = scenario(relay)
|
||||
self.assertGreater(f['reload_lines'], 400) # would drain a 400-line bucket
|
||||
e = f['err']
|
||||
for k in KEYS:
|
||||
self.assertIn(k, e, k)
|
||||
# the whole dump, line for line (blank lines are dropped by design)
|
||||
got = [l[len(relay.PREFIX):] for l in e.split('\n') if l]
|
||||
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
|
||||
|
||||
def test_dump_is_forwarded_in_full_even_with_an_empty_bucket(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in ('benign %d' % i for i in range(10)):
|
||||
r._emit(l)
|
||||
self.assertEqual(out.getvalue(), '')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n'), len([d for d in DUMP if d.strip()]))
|
||||
self.assertEqual(r.suppressed, 10)
|
||||
|
||||
def test_reload_noise_is_collapsed_not_drained(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for k in range(10):
|
||||
for l in reload_block(100 * (k + 1)):
|
||||
r._emit(l)
|
||||
self.assertEqual(r.tokens, 400.0) # bucket untouched by 1000+ noise lines
|
||||
self.assertEqual(r.suppressed, 0)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
summ = [l for l in out.getvalue().split('\n') if 'reload/start ok' in l]
|
||||
self.assertEqual(len(summ), 10) # exactly one per reload
|
||||
self.assertNotIn('Proxy be_', out.getvalue())
|
||||
self.assertNotIn('parsing [', out.getvalue())
|
||||
self.assertLess(out.getvalue().count('\n'), 15)
|
||||
|
||||
def test_suppressed_lines_are_summarised_per_minute(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(105):
|
||||
r._emit('flapping server state %d' % i)
|
||||
self.assertNotIn('[relay] suppressed', out.getvalue())
|
||||
t[0] += 61
|
||||
r.tick()
|
||||
self.assertRegex(out.getvalue(), r'\[relay\] suppressed 100 lines')
|
||||
self.assertEqual(r.suppressed, 0)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
|
||||
def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr(self):
|
||||
# docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "<N>" is NOT parsed
|
||||
f = scenario(relay)
|
||||
self.assertIn('[relay] reload/start ok', f['out'])
|
||||
self.assertNotIn('Thread', f['out'])
|
||||
self.assertNotIn('Current worker', f['out'])
|
||||
self.assertNotIn('[ALERT]', f['out'])
|
||||
self.assertNotIn('reload/start ok', f['err'])
|
||||
for text in (f['out'], f['err']):
|
||||
for l in text.split('\n'):
|
||||
self.assertFalse(l.startswith('<') or l.startswith(relay.PREFIX + '<'), l)
|
||||
|
||||
def test_clean_worker_exit_is_not_crash_evidence(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit('[WARNING] (1) : Former worker (5) exited with code 0 (Exit)')
|
||||
r._emit('[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)')
|
||||
self.assertEqual(err.getvalue().count('\n'), 1)
|
||||
self.assertIn('code 143', err.getvalue())
|
||||
|
||||
def test_crash_classes_never_limited(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
|
||||
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
|
||||
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)',
|
||||
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy',
|
||||
'[WARNING] (6328) : All workers exited. Exiting... (134)',
|
||||
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
'* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0']
|
||||
for l in evidence:
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n'), len(evidence))
|
||||
|
||||
FORGED = [
|
||||
'GET /FATAL/SIGSEGV/segfault HTTP/1.1',
|
||||
'[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout',
|
||||
'[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN',
|
||||
'[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)',
|
||||
'Host: Current worker (1) exited with code 134 (Aborted)',
|
||||
'Host: [ALERT] (1) : forged alert',
|
||||
'x-forwarded-for: Thread 1 is about to kill the process (pid 1).',
|
||||
'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1',
|
||||
'ha_panic watchdog FATAL SIGABRT',
|
||||
'backend ALERT_Current worker_exited with code 134',
|
||||
'[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN',
|
||||
]
|
||||
|
||||
def test_forged_keywords_stay_rate_limited(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in self.FORGED * 50:
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertEqual(r.suppressed, len(self.FORGED) * 50)
|
||||
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget
|
||||
# a forged header in the middle of normal traffic cannot open a block either
|
||||
r._emit('proxy: Thread 1 is about to kill the process (pid 1).')
|
||||
r._emit('free text that would be a dump line')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
|
||||
def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit("GET /Loading success HTTP/1.1")
|
||||
self.assertEqual(r.routine, 0)
|
||||
self.assertIn('Loading success', out.getvalue())
|
||||
|
||||
def test_flood_of_dump_headers_is_capped_by_the_window_budget(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(20000):
|
||||
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
|
||||
r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0')
|
||||
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
|
||||
n = err.getvalue().count('\n')
|
||||
self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently
|
||||
t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again
|
||||
before = err.getvalue().count('\n')
|
||||
r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)')
|
||||
self.assertEqual(err.getvalue().count('\n'), before + 1)
|
||||
|
||||
def test_bound_closed_block_cannot_rearm_during_cooldown(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 5):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertFalse(r.in_dump)
|
||||
r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header
|
||||
self.assertFalse(r.in_dump)
|
||||
r._emit('call trace line after the second header')
|
||||
self.assertNotIn('call trace line after', err.getvalue())
|
||||
t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again
|
||||
r._emit('Thread 1 is about to kill the process (pid 3).')
|
||||
self.assertTrue(r.in_dump)
|
||||
|
||||
def test_hostile_and_very_long_lines(self):
|
||||
r = self.make_relay()
|
||||
r.poll_once()
|
||||
self.append('A' * 500000 + '\n')
|
||||
self.append('\x1b]0;evil\x07 [ALERT] forged\x00\r\n')
|
||||
r.poll_once()
|
||||
outs = self.out.getvalue().split('\n')
|
||||
self.assertTrue(all(len(l) < relay.MAX_LINE + 200 for l in outs))
|
||||
self.assertNotIn('\x1b', self.out.getvalue())
|
||||
self.assertNotIn('\x00', self.out.getvalue())
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 2)
|
||||
|
||||
def test_unterminated_flood_is_bounded_in_memory(self):
|
||||
r = self.make_relay()
|
||||
r.poll_once()
|
||||
for _ in range(3):
|
||||
self.append('B' * 400000) # never a newline
|
||||
r.poll_once()
|
||||
self.assertLessEqual(len(r.pending), relay.MAX_PENDING)
|
||||
|
||||
def test_unclosed_dump_block_cannot_disable_the_limiter(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 500):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertLessEqual(err.getvalue().count('\n'), relay.DUMP_MAX_LINES + 3)
|
||||
self.assertGreaterEqual(r.suppressed, 400)
|
||||
|
||||
def test_dump_block_is_closed_by_time_bound(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
t[0] += relay.DUMP_MAX_SECONDS + 1
|
||||
r._emit('later benign line')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertNotIn('later benign', err.getvalue())
|
||||
|
||||
|
||||
# --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence --------
|
||||
TENANT_LINES = [
|
||||
"[ALERT] (10) : backend 'be_tenant1' has no server available!",
|
||||
"[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.",
|
||||
"[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.",
|
||||
"[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.",
|
||||
]
|
||||
|
||||
def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit(self.TENANT_LINES[0])
|
||||
t[0] += 0.001
|
||||
t[0] += 5.0
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l]
|
||||
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
|
||||
self.assertIn('exited with code 134', err.getvalue())
|
||||
self.assertNotIn('exited with code 134', out.getvalue())
|
||||
self.assertNotIn('has no server available', err.getvalue())
|
||||
|
||||
def test_tenant_lines_are_their_own_rate_limited_stdout_class(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)])
|
||||
self.assertEqual(err.getvalue(), '') # never PRIORITY 3
|
||||
self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2)
|
||||
self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped
|
||||
self.assertEqual(r.tokens, 400.0) # general bucket untouched
|
||||
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched
|
||||
self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET)
|
||||
self.assertGreater(r.tenant_suppressed, 1000)
|
||||
t[0] += relay.SUMMARY_EVERY + 1
|
||||
r.tick()
|
||||
self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines')
|
||||
self.assertEqual(r.tenant_suppressed, 0)
|
||||
|
||||
def test_other_alerts_are_not_crash_evidence(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit('[ALERT] (12) : config : unable to load')
|
||||
r._emit("[ALERT] (12) : backend 'x' has no server available!")
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr
|
||||
|
||||
|
||||
def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop
|
||||
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
|
||||
self.assertEqual(r.budget_left, 0)
|
||||
before = err.getvalue().count('\n')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
|
||||
|
||||
def test_rotated_backup_is_mode_0640(self):
|
||||
r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0)
|
||||
os.chmod(self.path, 0o644)
|
||||
self.append('x' * 100 + '\n')
|
||||
r.rotate_if_needed()
|
||||
self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640)
|
||||
|
||||
# shared helpers for the file-driven tests
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
|
||||
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
|
||||
open(self.path, 'w').close()
|
||||
self.out = io.StringIO()
|
||||
|
||||
def make_relay(self):
|
||||
return relay.Relay(self.path, self.out, 1000, 120, 10 ** 6, 3, clock=lambda: 0.0)
|
||||
|
||||
def append(self, text):
|
||||
with open(self.path, 'a') as fh:
|
||||
fh.write(text)
|
||||
|
||||
|
||||
# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------
|
||||
|
||||
def _scenario_ok(mod):
|
||||
f = scenario(mod)
|
||||
got = [l[len(mod.PREFIX):] for l in f['err'].split('\n') if l]
|
||||
crash_ok = all(k in f['err'] for k in KEYS) and got == [d.rstrip() for d in DUMP if d.strip()]
|
||||
collapse_ok = f['out'].count('reload/start ok') == 4 and 'Proxy be_' not in f['out'] and f['bucket'] > 300
|
||||
prio_ok = 'Thread' not in f['out'] and 'reload/start ok' not in f['err']
|
||||
return crash_ok, collapse_ok, prio_ok
|
||||
|
||||
|
||||
class MutantTests(unittest.TestCase):
|
||||
SRC = os.path.join(HERE, 'haproxy-stderr-relay.py')
|
||||
|
||||
def load_mutant(self, old, new):
|
||||
with open(self.SRC) as fh:
|
||||
src = fh.read()
|
||||
self.assertIn(old, src, 'mutation target vanished: update the test')
|
||||
mutated = src.replace(old, new, 1)
|
||||
self.assertNotEqual(mutated, src)
|
||||
d = tempfile.mkdtemp(prefix='hap-relay-mutant-') # private copy, the real file is untouched
|
||||
path = os.path.join(d, 'mutant.py')
|
||||
with open(path, 'w') as fh:
|
||||
fh.write(mutated)
|
||||
sp = importlib.util.spec_from_file_location('mutant', path)
|
||||
m = importlib.util.module_from_spec(sp)
|
||||
sp.loader.exec_module(m)
|
||||
return m
|
||||
|
||||
def test_unmutated_copy_passes_everything(self):
|
||||
self.assertEqual(_scenario_ok(relay), (True, True, True))
|
||||
|
||||
def test_mutant_limiter_applied_to_crash_lines(self):
|
||||
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:')
|
||||
crash_ok, _, _ = _scenario_ok(m)
|
||||
self.assertFalse(crash_ok)
|
||||
|
||||
def test_mutant_dump_block_not_tracked(self):
|
||||
m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:')
|
||||
crash_ok, _, _ = _scenario_ok(m)
|
||||
self.assertFalse(crash_ok)
|
||||
|
||||
def test_mutant_collapse_disabled(self):
|
||||
m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:')
|
||||
_, collapse_ok, _ = _scenario_ok(m)
|
||||
self.assertFalse(collapse_ok)
|
||||
|
||||
def test_mutant_crash_lines_on_stdout(self):
|
||||
m = self.load_mutant('self._write(self.err, line)', 'self._write(self.out, line)')
|
||||
_, _, prio_ok = _scenario_ok(m)
|
||||
self.assertFalse(prio_ok)
|
||||
|
||||
def test_mutant_dump_bound_removed(self):
|
||||
m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ')
|
||||
err = io.StringIO()
|
||||
r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(m.DUMP_MAX_LINES if False else 800):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block...
|
||||
|
||||
def _forged_run(self, mod):
|
||||
err = io.StringIO()
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in CapturedScenarioTests.FORGED * 20:
|
||||
r._emit(l)
|
||||
return err.getvalue()
|
||||
|
||||
def test_forged_run_clean_on_real_module(self):
|
||||
self.assertEqual(self._forged_run(relay), '')
|
||||
|
||||
def test_mutant_anchors_removed(self):
|
||||
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):',
|
||||
"if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):")
|
||||
self.assertNotEqual(self._forged_run(m), '')
|
||||
|
||||
def _flood(self, mod):
|
||||
err, t = io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(5000):
|
||||
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
|
||||
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
|
||||
return err.getvalue().count('\n')
|
||||
|
||||
def test_flood_capped_on_real_module(self):
|
||||
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
|
||||
def test_mutant_budget_removed(self):
|
||||
m = self.load_mutant('if self._budget(now, kind):', 'if True:')
|
||||
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
|
||||
def _rearm(self, mod):
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO())
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 5):
|
||||
r._emit('x %d' % i)
|
||||
r._emit('Thread 1 is about to kill the process (pid 2).')
|
||||
return r.in_dump
|
||||
|
||||
def test_mutant_cooldown_removed(self):
|
||||
self.assertFalse(self._rearm(relay))
|
||||
m = self.load_mutant(' and now >= self.cooldown_until:', ':')
|
||||
self.assertTrue(self._rearm(m))
|
||||
|
||||
def test_mutant_summary_removed(self):
|
||||
m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines")
|
||||
out, t = io.StringIO(), [0.0]
|
||||
r = m.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=io.StringIO())
|
||||
for i in range(105):
|
||||
r._emit('x %d' % i)
|
||||
t[0] += 61
|
||||
r.tick()
|
||||
self.assertNotIn('[relay] suppressed', out.getvalue())
|
||||
|
||||
|
||||
def _tenant_then_panic_ok(self, mod):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!")
|
||||
t[0] += 0.001
|
||||
t[0] += 5.0
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l]
|
||||
return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue()
|
||||
|
||||
def test_tenant_then_panic_clean_on_real_module(self):
|
||||
self.assertTrue(self._tenant_then_panic_ok(relay))
|
||||
|
||||
def test_mutant_wide_alert_exemption_restored(self):
|
||||
m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker")
|
||||
self.assertFalse(self._tenant_then_panic_ok(m))
|
||||
|
||||
def test_mutant_tenant_class_removed(self):
|
||||
m = self.load_mutant('if TENANT_RE.match(line):', 'if False:')
|
||||
self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m))
|
||||
|
||||
def _tenant_stdout_ok(self, mod):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
for i in range(1200):
|
||||
r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout")
|
||||
return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2
|
||||
|
||||
def test_mutant_shared_budget(self):
|
||||
m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'")
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(m.CRASH_BUDGET + 500):
|
||||
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
|
||||
before = err.getvalue().count('\n')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH)."""
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh')
|
||||
|
||||
PIDOF = """#!/bin/sh
|
||||
# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds
|
||||
[ -f "$STATE/started" ] || exit 1
|
||||
start=$(cat "$STATE/started")
|
||||
now=$(date +%s.%N)
|
||||
awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}'
|
||||
"""
|
||||
PYTHON = """#!/bin/sh
|
||||
# stand-in for ensure_haproxy.py: stamp the start, record it
|
||||
date +%s.%N > "$STATE/started"
|
||||
date +%s.%N >> "$STATE/starts"
|
||||
exit ${ENSURE_RC:-0}
|
||||
"""
|
||||
|
||||
|
||||
class SupervisorTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.d = tempfile.mkdtemp(prefix='hap-sup-')
|
||||
self.state = os.path.join(self.d, 'state')
|
||||
os.mkdir(self.state)
|
||||
for name, body in (('pidof', PIDOF), ('python', PYTHON)):
|
||||
p = os.path.join(self.d, name)
|
||||
with open(p, 'w') as fh:
|
||||
fh.write(body)
|
||||
os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR)
|
||||
|
||||
def run_sup(self, seconds, **env):
|
||||
e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state)
|
||||
e.setdefault('LIFE', '0')
|
||||
e.update(env)
|
||||
try:
|
||||
subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
return 'exited'
|
||||
except subprocess.TimeoutExpired:
|
||||
return 'running' # still looping when the clock ran out = alive
|
||||
|
||||
def starts(self):
|
||||
try:
|
||||
with open(os.path.join(self.state, 'starts')) as fh:
|
||||
return [float(x) for x in fh.read().split()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def test_non_integer_interval_does_not_kill_the_supervisor(self):
|
||||
self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running')
|
||||
self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying
|
||||
|
||||
def test_garbage_interval_and_max_fall_back(self):
|
||||
self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running')
|
||||
self.assertGreaterEqual(len(self.starts()), 1)
|
||||
|
||||
def test_crash_loop_backs_off_2_5_10_capped(self):
|
||||
# haproxy dies immediately after every successful start (LIFE=0) -> quick deaths
|
||||
self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3',
|
||||
HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8',
|
||||
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
|
||||
s = self.starts()
|
||||
gaps = [round(b - a, 1) for a, b in zip(s, s[1:])]
|
||||
self.assertGreaterEqual(len(s), 4, gaps)
|
||||
# gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4
|
||||
self.assertLess(gaps[0], gaps[1] + 0.3, gaps)
|
||||
self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps)
|
||||
self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps)
|
||||
self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps)
|
||||
|
||||
def test_stable_haproxy_resets_the_backoff(self):
|
||||
# alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth
|
||||
self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3',
|
||||
HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8',
|
||||
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
|
||||
s = self.starts()
|
||||
gaps = [b - a for a, b in zip(s, s[1:])]
|
||||
self.assertGreaterEqual(len(s), 3, gaps)
|
||||
self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top
|
||||
|
||||
def test_running_haproxy_is_left_alone(self):
|
||||
with open(os.path.join(self.state, 'started'), 'w') as fh:
|
||||
fh.write(str(time.time()))
|
||||
self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running')
|
||||
self.assertEqual(self.starts(), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Executable
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/verify-published-tags.sh (final publish gate).
|
||||
|
||||
CI run 5381 failed because buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'`
|
||||
and prints the whole inspect text, while buildx 0.38 prints a bare digest. A stub
|
||||
`imagetools` emulates BOTH output styles (plus `--raw`), so no registry or Docker
|
||||
is needed. Held invariants:
|
||||
- the 0.28-style (full text) and 0.38-style (bare digest) outputs both verify
|
||||
- a reported digest that disagrees with the sha256 of the raw manifest fails
|
||||
- a missing tag, a platform mismatch, or different digests across registries /
|
||||
tags fail the job
|
||||
- build-push.yaml's verify step uses the script, and release/* + rc-* tags
|
||||
resolve to ONLY <version>-rc and <sha> (never latest)
|
||||
|
||||
Running: python3 scripts/test-verify-published-tags.py (needs jq)
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
SCRIPT = os.path.join(HERE, 'verify-published-tags.sh')
|
||||
WORKFLOW = os.path.join(ROOT, '.gitea', 'workflows', 'build-push.yaml')
|
||||
|
||||
STUB = r'''#!/usr/bin/env python3
|
||||
import hashlib, json, os, sys
|
||||
a = sys.argv[1:]
|
||||
mode = os.environ['STUB_MODE'] # v028 | v038
|
||||
table = json.load(open(os.environ['STUB_TABLE'])) # ref -> {"raw": str, "lie": optional digest}
|
||||
assert a[0] == 'inspect'
|
||||
ref = [x for x in a[1:] if not x.startswith('-') and not x.startswith('{{')][0]
|
||||
ent = table.get(ref)
|
||||
if ent is None:
|
||||
sys.stderr.write('ERROR: %s: not found\n' % ref); sys.exit(1)
|
||||
raw = ent['raw'].encode() + (b'\n' if ent.get('nl') else b'')
|
||||
digest = ent.get('lie') or 'sha256:' + hashlib.sha256(ent['raw'].encode()).hexdigest()
|
||||
if '--raw' in a:
|
||||
sys.stdout.buffer.write(raw); sys.exit(0)
|
||||
if '--format' in a:
|
||||
if mode == 'v038':
|
||||
print(digest)
|
||||
else: # 0.28: --format ignored, whole text printed
|
||||
print('Name: ' + ref); print('MediaType: application/vnd.oci.image.index.v1+json')
|
||||
print('Digest: ' + digest)
|
||||
sys.exit(0)
|
||||
print('Name: ' + ref)
|
||||
print('MediaType: application/vnd.oci.image.index.v1+json')
|
||||
print('Digest: ' + digest)
|
||||
print(' ')
|
||||
print('Manifests: ')
|
||||
for m in json.loads(ent['raw'])['manifests']:
|
||||
print(' Name: %s@%s' % (ref, m['digest']))
|
||||
print(' Digest: sha256:' + 'f' * 64) # indented: must never be mistaken for the index digest
|
||||
print(' Platform: %s/%s' % (m['platform']['os'], m['platform']['architecture']))
|
||||
'''
|
||||
|
||||
|
||||
def index(archs, salt=''):
|
||||
ms = [{'mediaType': 'application/vnd.oci.image.manifest.v1+json',
|
||||
'digest': 'sha256:' + hashlib.sha256((x + salt).encode()).hexdigest(), 'size': 1,
|
||||
'platform': {'architecture': x, 'os': 'linux'}} for x in archs]
|
||||
ms.append({'mediaType': 'application/vnd.oci.image.manifest.v1+json',
|
||||
'digest': 'sha256:' + '7' * 64, 'size': 1,
|
||||
'platform': {'architecture': 'unknown', 'os': 'unknown'},
|
||||
'annotations': {'vnd.docker.reference.type': 'attestation-manifest'}})
|
||||
return json.dumps({'schemaVersion': 2, 'mediaType': 'application/vnd.oci.image.index.v1+json',
|
||||
'manifests': ms}, separators=(',', ':'))
|
||||
|
||||
|
||||
REPOS = ['reg.one/a/img', 'reg.two/b/img']
|
||||
TAGS = ['1.2-rc', 'abc123']
|
||||
|
||||
|
||||
class VerifyTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.stub = os.path.join(self.tmp.name, 'stub')
|
||||
open(self.stub, 'w').write(STUB)
|
||||
os.chmod(self.stub, os.stat(self.stub).st_mode | stat.S_IEXEC)
|
||||
self.table = os.path.join(self.tmp.name, 'table.json')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def table_for(self, raw, **over):
|
||||
t = {'%s:%s' % (r, g): {'raw': raw} for r in REPOS for g in TAGS}
|
||||
for k, v in over.items():
|
||||
ref = k.replace('__', ':').replace('_', '/', 0)
|
||||
t[ref] = v
|
||||
return t
|
||||
|
||||
def run_script(self, table, mode, platforms='linux/amd64,linux/arm64', tags=TAGS):
|
||||
json.dump(table, open(self.table, 'w'))
|
||||
args = [SCRIPT, '--platforms', platforms]
|
||||
for r in REPOS:
|
||||
args += ['--repo', r]
|
||||
for t in tags:
|
||||
args += ['--tag', t]
|
||||
env = dict(os.environ, IMAGETOOLS=self.stub, STUB_MODE=mode, STUB_TABLE=self.table)
|
||||
return subprocess.run(args, env=env, capture_output=True, text=True, timeout=60)
|
||||
|
||||
def test_buildx_028_full_text_output_verifies(self):
|
||||
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v028')
|
||||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||||
self.assertIn('verified:', r.stdout)
|
||||
self.assertNotIn('bad digest', r.stdout)
|
||||
|
||||
def test_buildx_038_bare_digest_output_verifies(self):
|
||||
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v038')
|
||||
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
|
||||
|
||||
def test_trailing_newline_on_raw_output_tolerated(self):
|
||||
raw = index(['amd64', 'arm64'])
|
||||
t = {'%s:%s' % (r, g): {'raw': raw, 'nl': True} for r in REPOS for g in TAGS}
|
||||
self.assertEqual(self.run_script(t, 'v028').returncode, 0)
|
||||
|
||||
def test_amd64_only_publish_matches_platforms(self):
|
||||
raw = index(['amd64'])
|
||||
self.assertEqual(self.run_script(self.table_for(raw), 'v028', 'linux/amd64').returncode, 0)
|
||||
r = self.run_script(self.table_for(raw), 'v028', 'linux/amd64,linux/arm64')
|
||||
self.assertEqual(r.returncode, 1)
|
||||
self.assertIn('platform mismatch', r.stdout)
|
||||
|
||||
def test_missing_tag_fails(self):
|
||||
t = self.table_for(index(['amd64', 'arm64']))
|
||||
del t['reg.two/b/img:abc123']
|
||||
r = self.run_script(t, 'v038')
|
||||
self.assertEqual(r.returncode, 1)
|
||||
self.assertIn('MISSING', r.stdout)
|
||||
|
||||
def test_different_digest_across_registries_fails(self):
|
||||
t = self.table_for(index(['amd64', 'arm64']))
|
||||
t['reg.two/b/img:1.2-rc'] = {'raw': index(['amd64', 'arm64'], salt='other')}
|
||||
for mode in ('v028', 'v038'):
|
||||
r = self.run_script(t, mode)
|
||||
self.assertEqual(r.returncode, 1, mode)
|
||||
self.assertIn('digest mismatch', r.stdout)
|
||||
|
||||
def test_reported_digest_disagreeing_with_raw_manifest_fails(self):
|
||||
t = self.table_for(index(['amd64', 'arm64']))
|
||||
t['reg.one/a/img:1.2-rc']['lie'] = 'sha256:' + '0' * 64
|
||||
for mode in ('v028', 'v038'):
|
||||
r = self.run_script(t, mode)
|
||||
self.assertEqual(r.returncode, 1, mode)
|
||||
self.assertIn('cross-check FAILED', r.stdout)
|
||||
|
||||
def test_garbage_manifest_fails(self):
|
||||
t = self.table_for('not json')
|
||||
self.assertEqual(self.run_script(t, 'v038').returncode, 1)
|
||||
|
||||
|
||||
class WorkflowTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import yaml
|
||||
cls.src = open(WORKFLOW).read()
|
||||
cls.doc = yaml.safe_load(cls.src)
|
||||
cls.steps = cls.doc['jobs']['Build-and-Push']['steps']
|
||||
|
||||
def step(self, name):
|
||||
return [s for s in self.steps if s.get('name') == name][0]
|
||||
|
||||
def test_verify_step_uses_script_not_format_digest(self):
|
||||
run = self.step('Verify published tags in every registry')['run']
|
||||
self.assertIn('scripts/verify-published-tags.sh', run)
|
||||
self.assertNotIn('{{.Manifest.Digest}}', run)
|
||||
|
||||
def test_run_blocks_are_valid_bash(self):
|
||||
for s in self.steps:
|
||||
if 'run' in s:
|
||||
p = subprocess.run(['bash', '-n'], input=s['run'], text=True, capture_output=True)
|
||||
self.assertEqual(p.returncode, 0, '%s: %s' % (s.get('name'), p.stderr))
|
||||
|
||||
def test_release_and_rc_never_publish_latest(self):
|
||||
run = self.step('Resolve tags')['run']
|
||||
m = re.search(r'refs/heads/release/\*\|refs/tags/rc-\*\)\s*\n\s*tags="([^"]*)"', run)
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m.group(1), '${VERSION}-rc ${SHA}')
|
||||
main = re.search(r'refs/heads/main\)\s*\n\s*tags="([^"]*)"', run)
|
||||
self.assertIsNotNone(main)
|
||||
self.assertEqual(main.group(1), 'latest ${VERSION} ${SHA}')
|
||||
# latest appears in no other tag assignment
|
||||
self.assertEqual(len(re.findall(r'tags="[^"]*latest', run)), 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
# Final publish gate: every <repo>:<tag> must resolve, carry exactly the
|
||||
# platforms the assemble step reported, and have ONE index digest across all
|
||||
# tags and all registries. Any miss or mismatch exits 1.
|
||||
#
|
||||
# verify-published-tags.sh --platforms linux/amd64,linux/arm64 \
|
||||
# --repo <registry/path> [--repo ...] --tag <tag> [--tag ...]
|
||||
#
|
||||
# The index digest is computed in a buildx-VERSION-INDEPENDENT way. CI run 5381
|
||||
# showed buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'` and prints the
|
||||
# whole inspect text, while 0.38 prints a bare digest. So the digest comes from:
|
||||
# 1. sha256 of the raw manifest bytes (`inspect --raw`) - always, authoritative
|
||||
# (a content digest IS the sha256 of the manifest bytes);
|
||||
# 2. the strictly anchored `Digest:` line of the default text output, if present;
|
||||
# 3. the `--format '{{.Manifest.Digest}}'` output, ONLY if it is a bare digest
|
||||
# (0.28's full-text output is ignored here, not an error).
|
||||
# Every source that is available must agree, else the ref fails.
|
||||
# Platforms are read from the raw JSON (also version independent).
|
||||
#
|
||||
# Env: IMAGETOOLS overrides the command (default "docker buildx imagetools");
|
||||
# the CI step sets it to "$BUILDX imagetools"; the test suite uses a stub.
|
||||
set -uo pipefail
|
||||
|
||||
IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools}
|
||||
platforms=""; repos=(); tags=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--platforms) platforms=${2-}; shift 2 ;;
|
||||
--repo) repos+=("${2-}"); shift 2 ;;
|
||||
--tag) tags+=("${2-}"); shift 2 ;;
|
||||
*) echo "verify-published-tags: unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$platforms" ] && [ ${#repos[@]} -gt 0 ] && [ ${#tags[@]} -gt 0 ] \
|
||||
|| { echo "::error::verify-published-tags: --platforms, --repo and --tag are required"; exit 2; }
|
||||
command -v jq >/dev/null || { echo "::error::verify-published-tags: jq is required"; exit 2; }
|
||||
|
||||
want=$(printf '%s' "$platforms" | tr ',' '\n' | sort | tr '\n' ' ')
|
||||
work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
|
||||
digest_re='^sha256:[0-9a-f]{64}$'
|
||||
ref_digest=""; ref_name=""; fail=0
|
||||
|
||||
for r in "${repos[@]}"; do
|
||||
for t in "${tags[@]}"; do
|
||||
ref="${r}:${t}"
|
||||
raw="$work/raw"
|
||||
if ! timeout 90 $IMAGETOOLS inspect --raw "$ref" >"$raw" 2>"$work/err" || [ ! -s "$raw" ]; then
|
||||
echo "::error::MISSING or unreadable: ${ref}: $(head -c 300 "$work/err")"; fail=1; continue
|
||||
fi
|
||||
# Raw bytes as served, and with trailing newlines stripped (some buildx
|
||||
# versions append one when printing); the content digest is one of the two.
|
||||
h1="sha256:$(sha256sum <"$raw" | cut -d' ' -f1)"
|
||||
h2="sha256:$(printf '%s' "$(cat "$raw")" | sha256sum | cut -d' ' -f1)"
|
||||
|
||||
# Source 2: strictly anchored top-level "Digest:" line of the text output.
|
||||
textd=""
|
||||
if timeout 90 $IMAGETOOLS inspect "$ref" >"$work/text" 2>/dev/null; then
|
||||
mapfile -t lines < <(grep -E '^Digest:[[:space:]]+sha256:[0-9a-f]{64}[[:space:]]*$' "$work/text" | sed -E 's/^Digest:[[:space:]]+//; s/[[:space:]]+$//')
|
||||
[ ${#lines[@]} -eq 1 ] && textd=${lines[0]}
|
||||
fi
|
||||
# Source 3: bare --format output, only when it really is a bare digest.
|
||||
fmtd=""
|
||||
if out=$(timeout 90 $IMAGETOOLS inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null); then
|
||||
out=$(printf '%s' "$out" | tr -d '[:space:]')
|
||||
[[ "$out" =~ $digest_re ]] && fmtd=$out
|
||||
fi
|
||||
|
||||
d=""
|
||||
for cand in "$textd" "$fmtd"; do
|
||||
[ -n "$cand" ] || continue
|
||||
if [ "$cand" != "$h1" ] && [ "$cand" != "$h2" ]; then
|
||||
echo "::error::digest cross-check FAILED for ${ref}: reported ${cand}, raw manifest hashes to ${h1}"; fail=1; d="!"; break
|
||||
fi
|
||||
d=$cand
|
||||
done
|
||||
[ "$d" = "!" ] && continue
|
||||
[ -n "$d" ] || d=$h2
|
||||
[[ "$d" =~ $digest_re ]] || { echo "::error::bad digest for ${ref}: ${d}"; fail=1; continue; }
|
||||
if [ -z "$textd$fmtd" ]; then
|
||||
echo "::warning::no reported digest for ${ref}; using sha256 of the raw manifest"
|
||||
fi
|
||||
|
||||
if ! got=$(jq -r '.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"' "$raw" 2>/dev/null | sort | tr '\n' ' '); then
|
||||
echo "::error::unparseable manifest for ${ref}"; fail=1; continue
|
||||
fi
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "::error::platform mismatch for ${ref}: have [${got}] want [${want}]"; fail=1
|
||||
fi
|
||||
echo "ok ${ref} ${d} [${got}]"
|
||||
if [ -z "$ref_digest" ]; then ref_digest="$d"; ref_name="$ref"
|
||||
elif [ "$d" != "$ref_digest" ]; then
|
||||
echo "::error::digest mismatch: ${ref}=${d} vs ${ref_name}=${ref_digest}"; fail=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
[ "$fail" = 0 ] || { echo "::error::tag verification FAILED"; exit 1; }
|
||||
echo "verified: ${tags[*]} in ${#repos[@]} registries, digest ${ref_digest}, platforms ${platforms}"
|
||||
Reference in new issue
Block a user