Merge pull request 'HAProxy 3.2 LTS edge image (default for all new deployments)' (#20) from feat/haproxy-upgrade into main
HAProxy Manager Build and Push / Build-and-Push (push) Successful in 9m40s

This commit was merged in pull request #20.
This commit is contained in:
jknapp committed 2026-11-17 21:07:33 +00:00
commit 193d39e109
20 files changed
+2089 -70

No files matched your search

+97 -7
View File
@@ -1,9 +1,15 @@
name: HAProxy Manager Build and Push
run-name: ${{ gitea.actor }} pushed a change to main
run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }}
on:
push:
branches:
- main
# Release candidates: a push to release/* builds the SAME image but publishes
# ONLY :<VERSION>-rc and :<sha>, never :latest and never the plain :<VERSION>
# (see the "Resolve tags" step). Main keeps its behaviour.
- 'release/**'
tags:
- 'rc-*'
# Manual publish (e.g. after a [skip ci] merge, or to re-publish after a
# failed arm64 leg).
workflow_dispatch:
@@ -77,6 +83,30 @@ jobs:
id: ver
run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT"
# Publishing strategy. main: :latest + :<version> + :<sha> (unchanged).
# release/* branch or rc-* tag: :<version>-rc + :<sha> ONLY. A release candidate
# is deployed per host by explicit tag (durable per-host pin in WHP); :latest is
# moved to it by a later main publish only after the staged rollout succeeded.
# Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused
# rather than guessed at.
- name: Resolve tags
id: tags
env:
REF: ${{ gitea.ref }}
VERSION: ${{ steps.ver.outputs.version }}
SHA: ${{ gitea.sha }}
run: |
case "$REF" in
refs/heads/main)
tags="latest ${VERSION} ${SHA}" ;;
refs/heads/release/*|refs/tags/rc-*)
tags="${VERSION}-rc ${SHA}" ;;
*)
echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;;
esac
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
echo "publishing tags: ${tags}"
# amd64: required. Pushed untagged by digest to both registries (same
# digest in each); tags are only created by the assemble step below.
- name: Build amd64 (push by digest)
@@ -103,19 +133,79 @@ jobs:
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true
# Three tags per registry: :latest (moving), :<version> (human-readable
# release), :<sha> (immutable, guaranteed-unique rollback target).
- name: Assemble tag indexes
# imagetools needs a NEW buildx. Same pinned, sha256-verified binary as
# mirror-base-image.yaml (kept identical on purpose). The assemble step
# previously ran on whatever buildx the (possibly ephemeral) runner had
# (0.13.1 on the runner that hung in run 5375, 0.15.1 on the trusted one).
- name: Install pinned buildx
env:
BUILDX_VERSION: v0.28.0
BUILDX_SHA256: 696bc104bac3bb708eff1af3f8bbc09fda0fd88f5757c1f9b404a35117889224
run: |
set -euo pipefail
dir="${RUNNER_TEMP:-/tmp}/pinned-buildx"; mkdir -p "$dir"
curl -fsSL --retry 3 --max-time 120 -o "$dir/docker-buildx" \
"https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
echo "${BUILDX_SHA256} $dir/docker-buildx" | sha256sum -c -
chmod +x "$dir/docker-buildx"
"$dir/docker-buildx" version
echo "BUILDX=$dir/docker-buildx" >> "$GITHUB_ENV"
# main: three tags per registry: :latest (moving), :<version> (human-readable
# release), :<sha> (immutable, guaranteed-unique rollback target).
# release/* or rc-*: :<version>-rc and :<sha> only (see "Resolve tags").
# Hardened after run 5375 (assemble hung after pushing, TTY progress, old
# buildx): plain progress, pinned buildx, hard 10 minute step timeout.
- name: Assemble tag indexes
id: assemble
timeout-minutes: 10
env:
BUILDKIT_PROGRESS: plain
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outputs.digest }}
ARM64_OUTCOME: ${{ steps.arm64.outcome }}
VERSION: ${{ steps.ver.outputs.version }}
SHA: ${{ gitea.sha }}
TAGS: ${{ steps.tags.outputs.tags }}
run: |
scripts/assemble-multiarch-index.sh \
set -euo pipefail
tag_args=()
for t in $TAGS; do tag_args+=(--tag "$t"); done
# Use the pinned buildx for create AND the script's platform inspects.
export IMAGETOOLS="${BUILDX} imagetools"
out=$(scripts/assemble-multiarch-index.sh \
--amd64 "$AMD64_DIGEST" \
--arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
--repo ghcr.io/shadowdao/haproxy-manager-base \
--tag latest --tag "$VERSION" --tag "$SHA"
"${tag_args[@]}" | tee /dev/stderr)
platforms=$(printf '%s\n' "$out" | sed -n 's/^platforms=//p' | tail -n1)
[ -n "$platforms" ] || { echo "::error::assemble script reported no platforms line"; exit 1; }
if [ "$platforms" = "linux/amd64" ]; then
echo "::warning::SINGLE-ARCH publish (amd64 only) on purpose: arm64 outcome was '${ARM64_OUTCOME}'. Tags carry no arm64 manifest."
else
echo "multi-arch publish: ${platforms}"
fi
echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
# Final gate: every tag in every registry must resolve, carry the platforms
# the assemble step reported, and have ONE digest across all tags and both
# registries (they are built from the same sources). Any miss fails the job
# instead of trusting that the create calls "looked" fine.
- name: Verify published tags in every registry
timeout-minutes: 10
env:
BUILDKIT_PROGRESS: plain
TAGS: ${{ steps.tags.outputs.tags }}
PLATFORMS: ${{ steps.assemble.outputs.platforms }}
run: |
set -euo pipefail
# Version-independent digest logic lives in the script (tested by
# scripts/test-verify-published-tags.py); buildx 0.28 ignores --format.
export IMAGETOOLS="${BUILDX} imagetools"
tag_args=()
for t in $TAGS; do tag_args+=(--tag "$t"); done
scripts/verify-published-tags.sh \
--platforms "$PLATFORMS" \
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
--repo ghcr.io/shadowdao/haproxy-manager-base \
"${tag_args[@]}"
+4 -1
View File
@@ -14,7 +14,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- **WAF body inspection**: `python3 scripts/test-waf-body-inspection.py` - offline (runs in the image build); pins `option http-buffer-request`, the `X-WHP-WAF-Body` complete/partial stamp (strip -> stamp -> SPOE -> strip), `tune.bufsize` against the 100 ms SPOE fail-open budget and the 65535-byte SPOP frame cap, and Coraza's body limits in `coraza-spoa/config.yaml`. Run it after touching the Coraza block, `tune.bufsize`, or the SPOE engine.
- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired/<stamp>/` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/<primary>` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`.
- **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch").
- **Publish verify gate**: `python3 scripts/test-verify-published-tags.py` - offline (needs `jq`; NOT run in the image build); `scripts/verify-published-tags.sh` is the last build-push.yaml step: every tag in every registry must resolve, carry the assembled platforms, and share ONE index digest. The digest is the sha256 of `imagetools inspect --raw` cross-checked against the text `Digest:` line / bare `--format` output (buildx 0.28 ignores `--format` and prints full text; CI run 5381).
- **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash".
- **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines.
- **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`.
- **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md
@@ -178,4 +180,5 @@ re-reads it on reload, so a failed runtime command must degrade to
- **The :8000 API refuses peers other than 127.0.0.1/::1 and the container's default gateway** (`HAPROXY_MGR_API_ALLOWED_PEERS`, default `127.0.0.1,::1,gateway`; ports via `HAPROXY_MGR_PEER_FILTER_PORTS`, default `8000`). haproxy-manager shares `client-net` with every tenant container; host callers arrive from the gateway via docker-proxy. Only effective with the WHP `-p 127.0.0.1:8000:8000` publish (a 0.0.0.0 publish lets a tenant hairpin through the host and arrive from the gateway). A sidecar that must call :8000 directly needs its IP added
- gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat
- Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/`
- HAProxy is version 3.0.11
- HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101)
- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (dump blocks opened only by the anchored `Thread N is about to kill` header, `[ALERT] (PID) :`, `Current worker (N) exited with code N`; ANCHORED patterns, so keywords echoed from URLs/Host/server names never match) bypasses the bucket within a budget (1000 lines / 10 min, 60 s cooldown after a bound closes a block) and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `<N>` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay
+58 -26
View File
@@ -23,49 +23,78 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.licenses="MIT"
# haproxy is PINNED to a reviewed UPSTREAM version (HAPROXY_UPSTREAM), not to one
# Debian package revision. It was previously unpinned, so the binary could move
# under us at Debian's timing - an upstream release that rejected our config would
# have broken an unrelated commit's build, or worse, shipped an edge that refuses
# to start (see the `haproxy -c` gate below for why that matters: a config HAProxy
# rejects leaves the container Up with 80/443 unbound and /health still 200).
# haproxy comes from the UPSTREAM-maintained apt repo https://haproxy.debian.net
# (Vincent Bernat's builds, the Debian maintainer's own), suite
# `trixie-backports-<series>`, NOT from Debian trixie's archive.
#
# History: this used to pin the exact Debian version (3.0.11-1+deb13u3). Debian
# security then shipped 3.0.11-1+deb13u4 (same upstream 3.0.11, same arch set) and
# the exact pin silently froze the image on the OLDER, unpatched revision, and
# would hard-fail the build as soon as u3 left the archive. Debian security
# revisions of the SAME upstream version are exactly what we want to take, so:
# * an apt preference (Pin: version 3.0.11-*) makes apt install the NEWEST
# Debian revision of that upstream version on any architecture;
# * the build then FAILS unless the installed package AND `haproxy -v` both
# report that upstream version, so a major/minor bump (3.0 -> 3.1, 3.0.12)
# can never arrive by accident: that still needs this ARG changed and the
# `haproxy -c` gate below to pass.
# The pin and the gate compose: pinned = the version moves deliberately; the gate
# answers immediately whether the new binary still accepts fleet config.
# WHY (2026-11-16 forensics): Debian trixie ships 3.0.11 (+ security revisions).
# That build aborts via HAProxy's own watchdog inside the DNS resolver
# (resolv_process_responses / snr_resolution_cb, haproxy issue #3101) - whole
# edge down 15-52 s per abort on sdbees/whp01. The fix ("dns: bind the
# nameserver sockets to the initiating thread", "resolvers: make
# process_resolvers() single-threaded") is in 3.0.13+, 3.1.11+ and 3.2.6+;
# Debian trixie will not take it. See docs/go-port/haproxy-edge-upgrade-and-logging.md
# in the WHP repo for the version evidence and the rollout plan.
#
# To move to another upstream version: change HAPROXY_UPSTREAM, rebuild, let the
# gate verify. Check what Debian offers with: apt-cache madison haproxy
ARG HAPROXY_UPSTREAM=3.0.11
# Series: 3.2 (LTS, supported to 2030-Q2). Its trixie package is built with
# USE_QUIC=1 against the system OpenSSL 3.5.x NATIVE QUIC API (depends on
# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream
# has no trixie packages and would need a source build.)
#
# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 ..
# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any
# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian
# package version, HAPROXY_EXACT (default below), and the build:
# * pins apt to it (`Pin: version <full Debian version>`; NOTE a bare
# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1",
# and with a non-matching pin apt falls back to Debian's own 3.0.11 package);
# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v`
# reports the series (so an unreachable repo, which would leave Debian's
# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident);
# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c`
# gate below proves the binary accepts the fleet config.
# Taking a new patch is a DELIBERATE edit of the default below (or
# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never
# passes it, so a rebuild is reproducible.
# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat,
# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with
# signed-by, never fetched at build time. The fingerprint is asserted below.
ARG HAPROXY_UPSTREAM=3.2
ARG HAPROXY_EXACT=3.2.26-1~bpo13+1
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only; CI sets it only after probing the cache with the
# pinned CA). Unset = the upstream Debian repositories, exactly as before. The
# apt sources are restored before this RUN ends (restore FAILS the build if any
# reference to the cache is left), so no shipped layer carries the cache address.
# NOTE: CT130 does not proxy haproxy.debian.net (see the design doc); that one
# repo is always fetched directly.
# Canonical script + docs: cloud-apache-container (scripts/pkg-cache-repos.sh,
# docs/pkg-cache.md there); this repo carries byte-identical copies.
ARG PKG_CACHE=
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/haproxy-archive-keyring.gpg
RUN /usr/local/sbin/pkg-cache-repos.sh apt && \
apt update -y && apt dist-upgrade -y && \
printf 'Package: haproxy\nPin: version %s-*\nPin-Priority: 1001\n' "${HAPROXY_UPSTREAM}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
apt install -y --no-install-recommends gnupg ca-certificates && \
keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \
[ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \
[ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \
apt purge -y gnupg && \
echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \
apt update -y && \
pin="${HAPROXY_EXACT}" && \
printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \
rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \
deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \
case "${deb_ver}" in "${HAPROXY_UPSTREAM}"-*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not upstream ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}-" && \
echo "haproxy pin OK: upstream ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \
[ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \
case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \
haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \
if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \
mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \
echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \
/usr/local/sbin/pkg-cache-repos.sh restore && \
apt clean && rm -rf /var/lib/apt/lists/*
WORKDIR /haproxy
@@ -122,11 +151,14 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \
&& python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \
&& python3 /haproxy/scripts/test-default-splash.py \
&& python3 /haproxy/scripts/test-access-log-receiver.py \
&& python3 /haproxy/scripts/test-haproxy-stderr-relay.py \
&& python3 /haproxy/scripts/test-haproxy-supervisor.py \
&& python3 /haproxy/scripts/test-config-rollback.py \
&& python3 /haproxy/scripts/test-cert-write-safety.py \
&& python3 /haproxy/scripts/test-cert-scripts.py \
&& python3 /haproxy/scripts/test-waf-body-inspection.py \
&& python3 /haproxy/scripts/test-gunicorn-socket-timeouts.py \
&& python3 /haproxy/scripts/test-gunicorn-access-logging.py \
&& python3 /haproxy/scripts/validate-rendered-config.py
# Create log directories
RUN mkdir -p /var/log/haproxy && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
+18
View File
@@ -451,6 +451,8 @@ Covered by `scripts/test-default-splash.py` (template order/conditions, the
page, list generation) and the `splash-off` scenario of
`scripts/validate-rendered-config.py`.
> gunicorn access logs (:8080 none, :8000 bounded file, not stdout): see docs/logging.md.
## Access Log (in-container, since 2026.11.2)
HAProxy logs (`log /log.sock ... format rfc5424`, `hap_header.tpl`) to a UNIX
@@ -492,6 +494,22 @@ Rollout: image bump + recreate with the volume and sysctl (WHP's
are in the image, outside the `/etc/haproxy` named volume, so nothing can be
shadowed; `haproxy.cfg` is regenerated on start with the new `log` line.
## HAProxy stderr (panic dumps, [ALERT]s) - since 2026.11.4
HAProxy is started as `haproxy -W -db` so the master keeps a real stderr (with
the `daemon` keyword HAProxy points it at /dev/null, which discarded the
`[ALERT]` lines, `Current worker (N) exited with code 134` and the whole
watchdog/`ha_panic` thread dump). It is appended to
`/var/log/haproxy/haproxy-stderr.log` (the `haproxy-logs` volume, so it
survives a recreate) and `scripts/haproxy-stderr-relay.py` re-emits every line
on the container's stderr prefixed `haproxy-stderr: ` (rate-limited, file
rotated at 20 MiB x 3). `docker logs haproxy-manager` and whatever docker
logging driver the host configured therefore carry the crash text; the
per-request access log stays out of it. The in-container supervisor probes
with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` seconds (default 2, was 15) and
backs off to `HAPROXY_SUPERVISOR_MAX_BACKOFF` (30) only when a restart attempt
fails. Measured (3.2.26, empty config): crash to listeners back = 1.2-2.0 s.
## Default Backend Configuration
With the default splash on (above), unrouted hosts no longer reach this page; it still serves the blocked-IP and suspended pages, and serves unrouted hosts again when `HAPROXY_DEFAULT_SPLASH=off`.
+1 -1
View File
@@ -1 +1 @@
2026.11.3
2026.11.6
+43
View File
@@ -0,0 +1,43 @@
# Container logging policy (what reaches `docker logs`)
`docker logs haproxy-manager` (and the docker logging driver: json-file today,
journald -> Graylog under WHP PR #260) must carry only low-volume, high-value
lines. Per-request logs stay out of it.
| Source | Destination | Volume |
|---|---|---|
| HAProxy edge access log (1.5M lines/day) | `/var/log/haproxy/access.log` via `haproxy-log-receiver.py` (20 MiB cap, gzip generations) | NOT in `docker logs` |
| HAProxy master stderr (ALERT, crash dumps) | `haproxy-stderr.log` -> `haproxy-stderr-relay.py` -> stdout (benign, rate limited) / stderr (crash evidence) | rare |
| Supervisor / relay / init lines | stdout / stderr | rare |
| gunicorn :8080 default-page / blocked-IP / suspended-site | NO access log | none |
| gunicorn :8000 management API access log | `/var/log/haproxy/api-access.log` (haproxy-logs volume, mode 0640), rotated by `scripts/gunicorn-access-rotate.py` (copy-truncate, 20 MiB x 3) | NOT in `docker logs` |
| gunicorn error log (tracebacks, worker timeouts) | stderr (`--error-logfile -`) | rare |
## Why
Before this change both gunicorns ran with `--access-logfile -`, so each :8080
hit and each :8000 API call was one docker stdout line. With the journald
driver that is one journal entry per request: journald's default rate limit
(10000 messages / 30 s per sending service) can then suppress the very
crash-dump lines the relay emits, and Graylog would be flooded.
## Knobs
`HAPROXY_MGR_API_ACCESS_LOG` (path), `HAPROXY_API_ACCESS_LOG_MAX_BYTES`
(20971520), `HAPROXY_API_ACCESS_LOG_BACKUPS` (3), `HAPROXY_API_ACCESS_LOG_POLL`
(30 s; the live file can exceed the cap by at most one interval of API traffic).
## Not done
Failed-auth (401/403) lines are NOT mirrored to stdout; they are in the file
(grep the status column). Add a rate-limited mirror later if wanted.
## Tests
`python3 scripts/test-gunicorn-access-logging.py` (runs in the image build):
real gunicorn, a burst of default-page requests produces zero stdout lines,
API requests land in the file not stdout, handler exceptions still reach
stderr, and rotation is bounded.
Deploying this needs a NEW rc image built from `feat/haproxy-upgrade`
(VERSION stays 2026.11.5 until then).
+89 -16
View File
@@ -766,6 +766,92 @@ def is_process_running(process_name):
return True
return False
# ---------------------------------------------------------------------------
# Launching the HAProxy master (2026-11 forensics change)
#
# haproxy.cfg carries `daemon`, and in daemon mode HAProxy re-points the
# master's stdin/stdout/stderr at /dev/null after it has parsed the config
# (MEASURED on 3.0.11: /proc/<master>/fd/2 -> /dev/null). That discarded every
# [ALERT], the "exited with code 134" master line and the whole ha_panic()/
# watchdog thread dump of the 2026-11 aborts: only the core file survived.
#
# `-db` overrides `daemon` (MEASURED: foreground master, still master-worker,
# reload via the CLI keeps the same master pid), so the master keeps a real
# stderr. We append it to HAPROXY_STDERR_LOG (haproxy-logs volume, survives a
# container recreate) and scripts/haproxy-stderr-relay.py forwards it to the
# container's stderr (-> docker logging driver -> host journald/Graylog).
#
# The master is started as `sh -c '... &'` so it is re-parented to PID 1 exactly
# as the old daemon fork did (PID 1 reaps it; a Popen child would sit as a
# zombie in the API worker and is_process_running('haproxy') would keep
# reporting it alive).
# ---------------------------------------------------------------------------
HAPROXY_STDERR_LOG = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
try:
HAPROXY_START_WAIT = float(os.environ.get('HAPROXY_START_WAIT', '10'))
except ValueError:
HAPROXY_START_WAIT = 10.0
def _stderr_log_path():
"""HAPROXY_STDERR_LOG if its directory is writable, else a /tmp fallback
(a missing volume must never stop HAProxy from starting)."""
path = HAPROXY_STDERR_LOG
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
# 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable)
os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640))
return path
except OSError:
return '/tmp/haproxy-stderr.log'
def spawn_haproxy_master():
"""Start `haproxy -W -db` detached, stderr appended to the stderr log.
Returns a subprocess.CompletedProcess (returncode 0, .stderr = the new log
text) once HAProxy has loaded its configuration; raises
subprocess.CalledProcessError (.stderr = what HAProxy printed) if it exits
or never comes up within HAPROXY_START_WAIT seconds. Same contract the old
blocking `subprocess.run([... '-W' ...], check=True)` call sites relied on.
"""
cmd = ['haproxy', '-W', '-db', '-S', '/tmp/haproxy-cli,level,admin',
'-f', HAPROXY_CONFIG_PATH]
log_path = _stderr_log_path()
try:
offset = os.path.getsize(log_path)
except OSError:
offset = 0
subprocess.run(
['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; '
'"$@" </dev/null >>"$log" 2>&1 &', 'sh', log_path] + cmd,
check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
def new_text():
try:
with open(log_path, 'rb') as fh:
fh.seek(offset)
return fh.read().decode('utf-8', 'replace')
except OSError:
return ''
deadline = time.monotonic() + HAPROXY_START_WAIT
started = time.monotonic()
while time.monotonic() < deadline:
text = new_text()
if 'Loading success.' in text:
return subprocess.CompletedProcess(cmd, 0, '', text)
if time.monotonic() - started > 1.0 and not is_process_running('haproxy'):
break
time.sleep(0.1)
text = new_text()
if is_process_running('haproxy') and '[ALERT]' not in text:
# Slow to log (big config) but alive: treat as started.
return subprocess.CompletedProcess(cmd, 0, '', text)
raise subprocess.CalledProcessError(1, cmd, output='', stderr=text[-4000:])
# Initialize template engine
template_loader = jinja2.FileSystemLoader(TEMPLATE_DIR)
template_env = jinja2.Environment(loader=template_loader)
@@ -871,12 +957,7 @@ def reload_haproxy():
return jsonify({'status': 'success'}), 200
else:
# Start HAProxy if it's not running
result = subprocess.run(
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
check=True,
capture_output=True,
text=True
)
result = spawn_haproxy_master()
if result.returncode == 0:
print("HAProxy started successfully")
log_operation('start_haproxy', True)
@@ -3778,10 +3859,7 @@ def reload_haproxy_safely(backup_status=None):
else:
# HAProxy not running, start it
try:
result = subprocess.run(
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
check=True, capture_output=True, text=True
)
result = spawn_haproxy_master()
logger.info("HAProxy started successfully")
# Now - and only now - is this config known good.
promote_current_config_to_backup()
@@ -4114,12 +4192,7 @@ def start_haproxy():
return
# Configuration is valid, start HAProxy
result = subprocess.run(
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
check=True,
capture_output=True,
text=True
)
result = spawn_haproxy_master()
if result.returncode == 0:
logger.info("HAProxy started successfully")
log_operation('start_haproxy', True, 'HAProxy started successfully')
Binary file not shown.
+12
View File
@@ -34,6 +34,18 @@ def main():
haproxy_manager.logger.warning(
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
)
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
# before it vanished (the relay forwards the same text; this ties it to the
# supervisor event even if the relay is behind).
try:
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
fh.seek(0, 2)
fh.seek(max(0, fh.tell() - 1500))
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
for line in tail:
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
except OSError:
pass
# start_haproxy() validates the config (and regenerates it if invalid)
# before launching, and swallows its own errors, so it will not raise here.
haproxy_manager.start_haproxy()
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Bound the management-API (:8000) gunicorn access log file.
Why
---
Gunicorn's access log used to go to stdout (`--access-logfile -`), i.e. every
:8080 default-page / blocked-IP / suspended-site hit and every :8000 API call
became a `docker logs` line. With the journald docker log driver that is one
journal entry per request (journald's default rate limit, 10000 msgs / 30 s per
sending service, can then SUPPRESS the crash-dump lines the relay emits) and it
floods Graylog. Now:
* :8080 (default page) has NO access log at all;
* :8000 (API) writes its access log to API_ACCESS_LOG (default
/var/log/haproxy/api-access.log, in the haproxy-logs volume), NOT stdout;
* gunicorn ERROR logging (tracebacks, worker timeouts) stays on stderr.
This process keeps that file bounded: copy-truncate rotation at
HAPROXY_API_ACCESS_LOG_MAX_BYTES (default 20 MiB), HAPROXY_API_ACCESS_LOG_BACKUPS
generations (default 3), rotated copies mode 0640 (same policy as
haproxy-stderr.log). Truncating in place is safe because gunicorn opens the
file in append mode. Several gunicorn processes share the file, so renaming it
is not an option.
Bound: size is checked every HAPROXY_API_ACCESS_LOG_POLL seconds (default 30),
so the live file can exceed the limit only by what the API writes in one
interval (a few hundred bytes per call).
"""
import os
import shutil
import sys
import time
def _env_num(name, default, cast=int):
try:
v = cast(os.environ.get(name, default))
return v if v > 0 else default
except (TypeError, ValueError):
return default
def rotate_if_needed(path, max_bytes, backups):
"""Copy-truncate rotate. Returns True if it rotated."""
try:
if os.path.getsize(path) <= max_bytes:
return False
for i in range(backups, 1, -1):
src = '%s.%d' % (path, i - 1)
if os.path.exists(src):
os.replace(src, '%s.%d' % (path, i))
shutil.copyfile(path, path + '.1')
os.chmod(path + '.1', 0o640)
with open(path, 'r+b') as fh:
fh.truncate(0)
return True
except FileNotFoundError:
return False
except OSError as exc:
sys.stderr.write('gunicorn-access-rotate: rotate failed: %s\n' % exc)
return False
def main():
path = os.environ.get('HAPROXY_MGR_API_ACCESS_LOG', '/var/log/haproxy/api-access.log')
max_bytes = _env_num('HAPROXY_API_ACCESS_LOG_MAX_BYTES', 20 * 1024 * 1024)
backups = _env_num('HAPROXY_API_ACCESS_LOG_BACKUPS', 3)
poll = _env_num('HAPROXY_API_ACCESS_LOG_POLL', 30.0, float)
while True:
try:
rotate_if_needed(path, max_bytes, backups)
except Exception as exc: # never die; the supervisor loop restarts us anyway
sys.stderr.write('gunicorn-access-rotate: %r\n' % exc)
time.sleep(poll)
if __name__ == '__main__':
main()
+363
View File
@@ -0,0 +1,363 @@
#!/usr/bin/env python3
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
Why
---
HAProxy's master stderr carries what the access log never will: `[ALERT]`
lines, "Current worker (N) exited with code 134", and the complete
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
haproxy-logs volume, so it survives a container recreate). This process tails the
file and re-emits each line on ITS stderr, prefixed, which is the container's
stderr: `docker logs`, and through the docker logging driver the host journal
and Graylog.
Properties
----------
* Rare by nature: HAProxy only writes here on start/reload/server state
changes/crashes. The per-request access log is NOT here (it goes to the
receiver socket), so this does not feed the WHP AI monitor a firehose.
* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE
("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is
about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/
server name never matches) and every line of a dump block (from a panic/watchdog header to
the master's "All workers exited" line, bounded by
DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block
can open; blocks open only on the anchored "Thread N is about to kill" header)
are forwarded in full even when the bucket is empty, within a total budget of
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
and are counted in the suppressed summary). They go to STDERR.
* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!",
"Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints
the first at [ALERT]: every tenant container stop/crash/deploy makes them.
Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own
"[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI
monitor through `docker logs haproxy-manager`; rate-limiting them is intended.
Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend.
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
"Initializing new worker", "Loading success", ...) never touches the bucket:
it is counted and collapsed into ONE "[relay] reload/start ok: N routine
lines collapsed" line per reload (plus one on the minute tick for the late
"Proxy stopped" lines). The full text stays in the stderr file.
* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST
lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute
"[relay] suppressed N lines" summary.
* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and
stderr -> PRIORITY 3 (err) and does NOT parse "<N>" prefixes. So benign
lines (routine, rate-limited remainder, relay summaries) are written to
STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority.
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
Truncating is safe because HAProxy's fd is O_APPEND.
* Never blocks or affects HAProxy: it only reads a file.
Environment
-----------
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
HAPROXY_STDERR_RELAY_POLL (0.5 s)
"""
import os
import re
import shutil
import sys
import time
PREFIX = 'haproxy-stderr: '
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this
DUMP_MAX_SECONDS = 30.0
SUMMARY_EVERY = 60.0
# Every pattern is ANCHORED to HAProxy's own message structure (captured from
# real 3.2 crashes): a keyword inside a request path / Host header / server
# name that HAProxy echoes into a warning can never match, because the line
# prefix "[ALERT] (PID) :" is written by HAProxy itself.
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
CRASH_RE = re.compile(
r'^(?:'
r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
r'|Thread \d+ is about to kill the process'
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
r')')
# Tenant-driven state lines. WHP gives every tenant server `check`, so every
# tenant container stop/crash/OOM/deploy and every host-wide container restart
# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash
# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so
# they can neither reach priority 3 nor spend the general bucket / crash budgets.
TENANT_RE = re.compile(
r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:'
r"backend '[^']*' has no server available!"
r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)'
r'|Health check for (?:backup )?server \S+ '
r')')
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
# A new start/reload cycle also closes a dump block.
CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)')
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
ROUTINE_RE = re.compile(
r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:'
r'Proxy \S+ stopped \(cumulated conns'
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
r'|Former worker \(\d+\) exited with code 0\b'
r'|Reloading HAProxy|Initializing new worker|Loading success'
r'|path to executable is|haproxy version is'
r"|config: Can't load stats-file|'option forwardfor' ignored)")
LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success')
# Total exemption budget: even real-looking evidence cannot flood without bound.
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
CRASH_WINDOW = 600.0 # seconds
DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that
# ordinary evidence lines (worker exits, alerts) can never spend
TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute)
TENANT_PER_MIN = 30
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
def _env_num(name, default, cast=int):
try:
v = cast(os.environ.get(name, default))
return v if v > 0 else default
except (TypeError, ValueError):
return default
class Relay:
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None):
self.path = path
self.out = out # benign lines (docker journald: PRIORITY 6)
self.err = err if err is not None else out # crash lines (PRIORITY 3)
self.burst, self.per_min = burst, per_min
self.max_bytes, self.backups = max_bytes, backups
self.clock = clock
self.tokens = float(burst)
self.last = clock()
self.suppressed = 0
self.routine = 0 # collapsed routine lines since the last summary
self.summary_at = clock()
self.in_dump = False
self.dump_lines = 0
self.dump_since = 0.0
self.cooldown_until = 0.0
self.budget_left = CRASH_BUDGET
self.window_start = clock()
self.dump_budget_left = DUMP_BUDGET
self.dump_window_start = clock()
self.tenant_tokens = float(TENANT_BURST)
self.tenant_last = clock()
self.tenant_suppressed = 0
self.offset = None # None = first pass: start at EOF (do not replay history)
self.pending = ''
self._last_stream = None
def _take(self):
now = self.clock()
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
self.last = now
if self.tokens >= 1:
self.tokens -= 1
return True
return False
def _take_tenant(self):
now = self.clock()
self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0)
self.tenant_last = now
if self.tenant_tokens >= 1:
self.tenant_tokens -= 1
return True
return False
def _write(self, stream, text):
# keep the two streams in emission order: flush the other one first
if self._last_stream is not None and self._last_stream is not stream:
self._last_stream.flush()
stream.write(PREFIX + text + '\n')
self._last_stream = stream
def _flush_all(self):
for st in {id(self.out): self.out, id(self.err): self.err}.values():
try:
st.flush()
except Exception:
pass
def _summaries(self, force=False):
"""Per-minute summary of what the limiter / collapse withheld."""
now = self.clock()
if not force and now - self.summary_at < SUMMARY_EVERY:
return
self.summary_at = now
if self.routine:
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
if self.tenant_suppressed:
self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines '
'(rate limit; full text in the stderr file)' % self.tenant_suppressed)
self.tenant_suppressed = 0
if self.suppressed:
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
% self.suppressed)
self.suppressed = 0
def _budget(self, now, kind='crash'):
"""Take one unit of a budget; False once the window's budget is spent.
kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget)."""
if kind == 'dump':
if now - self.dump_window_start >= CRASH_WINDOW:
self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET
if self.dump_budget_left > 0:
self.dump_budget_left -= 1
return True
return False
if now - self.window_start >= CRASH_WINDOW:
self.window_start, self.budget_left = now, CRASH_BUDGET
if self.budget_left > 0:
self.budget_left -= 1
return True
return False
def _end_dump(self, now, why=None):
self.in_dump = False
if why: # closed by a bound, not by the master's end line: no re-arming for a while
self.cooldown_until = now + DUMP_COOLDOWN
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
def _emit(self, line):
now = self.clock()
if self.in_dump:
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
elif CYCLE_RE.match(line):
self._end_dump(now)
if self.in_dump or CRASH_RE.match(line):
is_dump = self.in_dump or bool(DUMP_START_RE.match(line))
kind = 'dump' if is_dump else 'crash'
if self._budget(now, kind):
# crash evidence: bypasses the bucket, the collapse and the summary
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
if self.in_dump:
self.dump_lines += 1
self._write(self.err, line)
if self.in_dump and DUMP_END_RE.match(line):
self._end_dump(now)
return
if self.in_dump: # budget spent mid-block: the normal limiter takes over
self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW))
if ROUTINE_RE.match(line):
self.routine += 1
if LOADED_RE.match(line): # one summary per start/reload
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
self._summaries()
return
if TENANT_RE.match(line):
if self._take_tenant():
self._write(self.out, line)
else:
self.tenant_suppressed += 1
self._summaries()
return
if self._take():
self._write(self.out, line)
else:
self.suppressed += 1
self._summaries()
@staticmethod
def _clean(line):
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
if len(line) > MAX_LINE:
line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line)
return line
def rotate_if_needed(self):
try:
if os.path.getsize(self.path) <= self.max_bytes:
return
for i in range(self.backups, 1, -1):
src = '%s.%d' % (self.path, i - 1)
if os.path.exists(src):
os.replace(src, '%s.%d' % (self.path, i))
shutil.copyfile(self.path, self.path + '.1')
os.chmod(self.path + '.1', 0o640)
with open(self.path, 'r+b') as fh:
fh.truncate(0)
self.offset = 0
except OSError as exc:
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
def poll_once(self):
try:
size = os.path.getsize(self.path)
except OSError:
if self.offset is None:
self.offset = 0 # file not created yet: whatever appears is new
return
if self.offset is None:
self.offset = size
return
if size < self.offset: # truncated by someone else
self.offset = 0
if size == self.offset:
return
with open(self.path, 'rb') as fh:
fh.seek(self.offset)
chunk = fh.read(1 << 20)
self.offset += len(chunk)
text = self.pending + chunk.decode('utf-8', 'replace')
lines = text.split('\n')
self.pending = lines.pop() # incomplete trailing line stays buffered
if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line
lines.append(self.pending)
self.pending = ''
for line in lines:
line = self._clean(line)
if line:
self._emit(line)
self._summaries()
self._flush_all()
def tick(self):
"""Idle-time housekeeping: emit the per-minute summaries even when no line arrives."""
self._summaries()
self._flush_all()
def main():
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
relay = Relay(path, sys.stdout,
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
err=sys.stderr)
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
sys.stdout.write('haproxy-stderr-relay: following %s\n' % path)
sys.stdout.flush()
ticks = 0
while True:
try:
relay.poll_once()
relay.tick()
ticks += 1
if ticks % 120 == 0:
relay.rotate_if_needed()
except Exception as exc: # never die: the supervisor loop restarts us anyway
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
time.sleep(poll)
if __name__ == '__main__':
main()
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
# In-container haproxy supervisor (started in the background by start-up.sh).
#
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
# policy never fires, and haproxy is down until the external host watchdog
# full-restarts the whole container minutes later (dropping every connection).
# This loop revives haproxy in place within one interval.
#
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
# most of it detection latency + start; the default is now 2 s (was 15).
#
# Two backoffs:
# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its
# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
# re-rendering/re-validating the same broken config every 2 s;
# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH
# seconds (default 10) of that start (a crash loop): extra wait before the
# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default
# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives
# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death
# after a long run is still revived within one interval.
#
# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5,
# "abc") falls back to the default instead of killing this subshell under set -e.
set -eo pipefail
_posint() { # _posint VALUE DEFAULT
case "$1" in
''|*[!0-9]*) echo "$2" ;;
*) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;;
esac
}
INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)"
MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)"
QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)"
STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)"
STEPS=""
for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do
v="$(_posint "$s" 0)"
[ "$v" -ge 1 ] && STEPS="${STEPS} ${v}"
done
[ -n "${STEPS}" ] || STEPS=" 2 5 10 30"
# step N (1-based, past the end = last), capped at MAX_BACKOFF
_step() {
local n="$1" i=0 v=0
for s in ${STEPS}; do
i=$((i + 1)); v="$s"
[ "${i}" -ge "${n}" ] && break
done
[ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}"
echo "${v}"
}
delay="${INTERVAL}"
last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet
quick=0 # consecutive deaths within QUICK_DEATH of a start
while true; do
sleep "${delay}"
if pidof -q haproxy; then
delay="${INTERVAL}"
if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then
quick=0; last_start=-1
fi
continue
fi
if [ "${last_start}" -ge 0 ]; then
up=$((SECONDS - last_start))
if [ "${up}" -lt "${QUICK_DEATH}" ]; then
quick=$((quick + 1))
wait_s="$(_step "${quick}")"
echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2
sleep "${wait_s}"
pidof -q haproxy && continue
elif [ "${up}" -ge "${STABLE}" ]; then
quick=0
fi
fi
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
delay="${INTERVAL}"
last_start="${SECONDS}"
else
delay=$(( delay * 2 ))
[ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}"
fi
done
+42 -17
View File
@@ -75,24 +75,49 @@ mkdir -p /var/log/haproxy /var/lib/haproxy
cron &
# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and
# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs,
# "exited with code 134", the watchdog/ha_panic thread dump) is appended to
# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's
# stderr so `docker logs` / the docker logging driver / Graylog get it. Started
# BEFORE init.py so the first start's lines are followed. Restarted within 2 s
# if it ever dies; it only reads a file, so it can never affect HAProxy.
(
while true; do
python /haproxy/scripts/haproxy-stderr-relay.py || true
sleep 2
done
) &
# gunicorn access logs (incident-prevention for the journald log driver).
# gunicorn used to log every request to stdout (`--access-logfile -`), so every
# :8080 default-page / blocked-IP hit and every :8000 API call became a docker
# log line -> journal entry -> Graylog line, and could trip journald's rate
# limit and suppress the relay's crash-dump lines. Now:
# :8080 no access log at all;
# :8000 access log to a size-bounded FILE in the haproxy-logs volume (mode
# 0640), rotated by scripts/gunicorn-access-rotate.py (copy-truncate,
# 20 MiB x 3). Set HAPROXY_MGR_API_ACCESS_LOG to relocate it.
# Gunicorn ERROR logging (--error-logfile -) stays on stderr -> `docker logs`.
API_ACCESS_LOG="${HAPROXY_MGR_API_ACCESS_LOG:-/var/log/haproxy/api-access.log}"
export HAPROXY_MGR_API_ACCESS_LOG="${API_ACCESS_LOG}"
mkdir -p "$(dirname "${API_ACCESS_LOG}")"
[ -e "${API_ACCESS_LOG}" ] || : > "${API_ACCESS_LOG}"
chmod 0640 "${API_ACCESS_LOG}" 2>/dev/null || true
(
while true; do
python /haproxy/scripts/gunicorn-access-rotate.py || true
sleep 2
done
) &
# Phase 1: container init
python /haproxy/scripts/init.py
# Phase 1.5: in-container haproxy supervisor.
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
# policy never fires, and haproxy is down until the external host watchdog
# full-restarts the whole container minutes later (dropping every connection).
# This loop revives haproxy in place within one interval. ensure_haproxy.py is
# idempotent — a cheap no-op whenever haproxy is already running.
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}"
(
while true; do
sleep "${HAPROXY_SUPERVISOR_INTERVAL}"
python /haproxy/scripts/ensure_haproxy.py 2>&1 || true
done
) &
# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh:
# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with
# backoff on failed restarts and on crash loops; knobs validated as integers).
/haproxy/scripts/haproxy-supervisor.sh &
# Phase 2: WSGI servers
# Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT
@@ -132,7 +157,7 @@ gunicorn \
--workers 1 --threads 4 --worker-class "${WORKER_CLASS}" \
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
--timeout 30 \
--access-logfile - --error-logfile - --log-level info \
--error-logfile - --log-level info \
--pythonpath /haproxy \
'haproxy_manager:default_app' &
@@ -144,6 +169,6 @@ exec gunicorn \
--workers "${API_WORKERS}" --threads 4 --worker-class "${WORKER_CLASS}" \
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
--timeout "${API_TIMEOUT}" \
--access-logfile - --error-logfile - --log-level info \
--access-logfile "${API_ACCESS_LOG}" --error-logfile - --log-level info \
--pythonpath /haproxy \
'haproxy_manager:app'
+7 -1
View File
@@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR)
# The module opens /var/log/haproxy-manager.log at import time via
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
# spawn_haproxy_master() appends the master's stderr here.
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
os.environ.setdefault('HAPROXY_START_WAIT', '3')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda fn, *a, **kw: _real_file_handler(
@@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
# haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN}
# reject any unusable file in $TEST_CERTS_DIR
cfg=""
start=0
while [ $# -gt 0 ]; do
case "$1" in -f) cfg="$2"; shift ;; esac
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
shift
done
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
@@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
fi
done
fi
# real `haproxy -W -db` prints this once the config is loaded
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
exit 0
""")
+9 -1
View File
@@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR)
# The module opens /var/log/haproxy-manager.log at import time via
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
# spawn_haproxy_master() appends the master's stderr here (default is the
# haproxy-logs volume path, not writable unprivileged).
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
os.environ.setdefault('HAPROXY_START_WAIT', '3')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda fn, *a, **kw: _real_file_handler(
@@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
cfg=""
start=0
while [ $# -gt 0 ]; do
case "$1" in -f) cfg="$2"; shift ;; esac
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
shift
done
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
exit 1
fi
# A real `haproxy -W -db` prints this once the config is loaded;
# spawn_haproxy_master() waits for it.
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
exit 0
""")
+177
View File
@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Tests: gunicorn access logging must not flood the container's stdout.
Context: with the journald docker log driver every stdout line is a journal
entry (default rate limit 10000 msgs/30 s per service could suppress crash-dump
lines) and, via Graylog, a stored message. start-up.sh therefore gives :8080 no
access log and sends :8000's to a bounded FILE; gunicorn errors stay on stderr.
Pins (static, from start-up.sh):
* no gunicorn invocation uses `--access-logfile -`;
* the :8080 invocation has no --access-logfile at all;
* the :8000 invocation logs access to ${API_ACCESS_LOG}, errors to `-`.
Pins (live, REAL gunicorn, same flag shapes):
* a burst of default-page requests (no access log) writes ZERO lines to
stdout and none to stderr;
* a burst of API requests writes ZERO stdout lines, the access file receives
them, and a handler exception still reaches STDERR (error logging intact);
* scripts/gunicorn-access-rotate.py keeps the file bounded (copy-truncate,
backups capped, rotated copy mode 0640).
Runs in the Docker build gate.
"""
import importlib.util
import os
import re
import socket
import stat
import subprocess
import sys
import tempfile
import textwrap
import time
import unittest
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
try:
import gunicorn # noqa: F401
except ImportError:
sys.stderr.write('SKIP: gunicorn not installed\n')
raise SystemExit(77)
spec = importlib.util.spec_from_file_location('rot', os.path.join(HERE, 'gunicorn-access-rotate.py'))
rot = importlib.util.module_from_spec(spec)
spec.loader.exec_module(rot)
APP = textwrap.dedent('''\
def app(environ, start_response):
if environ['PATH_INFO'] == '/boom':
raise RuntimeError('handler-exploded')
start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '2')])
return [b'ok']
''')
def free_port():
s = socket.socket()
s.bind(('127.0.0.1', 0))
p = s.getsockname()[1]
s.close()
return p
def start_script():
with open(os.path.join(HERE, 'start-up.sh')) as fh:
return fh.read()
class StaticFlags(unittest.TestCase):
def setUp(self):
self.src = start_script()
# Each invocation = from "gunicorn \" to its app target line.
self.inv = re.findall(r'^(?:exec )?gunicorn \\\n(?:.*\\\n)*?.*haproxy_manager:\w+.*$', self.src, re.M)
def test_two_invocations(self):
self.assertEqual(len(self.inv), 2, self.inv)
def test_no_stdout_access_log(self):
for i in self.inv:
self.assertNotRegex(i, r'--access-logfile\s+-(\s|$)')
def test_default_page_has_no_access_log(self):
d = [i for i in self.inv if '0.0.0.0:8080' in i]
self.assertEqual(len(d), 1)
self.assertNotIn('--access-logfile', d[0])
self.assertIn('--error-logfile -', d[0])
def test_api_access_log_is_a_file(self):
a = [i for i in self.inv if '0.0.0.0:8000' in i]
self.assertEqual(len(a), 1)
self.assertIn('--access-logfile "${API_ACCESS_LOG}"', a[0])
self.assertIn('--error-logfile -', a[0])
def test_rotator_started(self):
self.assertIn('gunicorn-access-rotate.py', self.src)
class Live(unittest.TestCase):
N_DEFAULT = 300
N_API = 100
def setUp(self):
self.dir = tempfile.mkdtemp(prefix='gunicorn-access-')
with open(os.path.join(self.dir, 'tapp.py'), 'w') as fh:
fh.write(APP)
def run_gunicorn(self, extra, n, boom=False):
port = free_port()
out = open(os.path.join(self.dir, 'out-%d' % port), 'w+')
err = open(os.path.join(self.dir, 'err-%d' % port), 'w+')
p = subprocess.Popen(
[sys.executable, '-m', 'gunicorn', '--bind', '127.0.0.1:%d' % port,
'--workers', '1', '--threads', '4', '--error-logfile', '-', '--log-level', 'info',
'--pythonpath', self.dir] + extra + ['tapp:app'],
stdout=out, stderr=err)
try:
for _ in range(100):
try:
urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=2).read()
break
except Exception:
time.sleep(0.2)
for _ in range(n):
urllib.request.urlopen('http://127.0.0.1:%d/x' % port, timeout=5).read()
if boom:
try:
urllib.request.urlopen('http://127.0.0.1:%d/boom' % port, timeout=5).read()
except Exception:
pass
time.sleep(0.5)
finally:
p.terminate()
p.wait(10)
out.seek(0)
err.seek(0)
o, e = out.read(), err.read()
out.close()
err.close()
return o, e
def test_default_page_burst_is_silent(self):
out, err = self.run_gunicorn([], self.N_DEFAULT)
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
self.assertNotIn('GET /x', err)
def test_api_burst_goes_to_file_not_stdout_and_errors_stay_on_stderr(self):
path = os.path.join(self.dir, 'api-access.log')
open(path, 'w').close()
out, err = self.run_gunicorn(['--access-logfile', path], self.N_API, boom=True)
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
self.assertNotIn('GET /x', err)
with open(path) as fh:
lines = fh.read().splitlines()
self.assertGreaterEqual(len([l for l in lines if 'GET /x' in l]), self.N_API)
self.assertIn('handler-exploded', err) # error logging intact
class Rotation(unittest.TestCase):
def test_bounded_and_mode(self):
d = tempfile.mkdtemp(prefix='gunicorn-rot-')
path = os.path.join(d, 'api-access.log')
for _ in range(12):
with open(path, 'ab') as fh:
fh.write(b'x' * 1000 + b'\n')
rot.rotate_if_needed(path, 4000, 3)
files = sorted(os.listdir(d))
self.assertLessEqual(len(files), 4, files) # live + 3 backups
self.assertLessEqual(os.path.getsize(path), 4000 + 1001)
self.assertEqual(stat.S_IMODE(os.stat(path + '.1').st_mode), 0o640)
self.assertFalse(os.path.exists(path + '.4'))
def test_missing_file_is_not_fatal(self):
self.assertFalse(rot.rotate_if_needed('/nonexistent/api-access.log', 10, 3))
if __name__ == '__main__':
unittest.main()
+614
View File
@@ -0,0 +1,614 @@
#!/usr/bin/env python3
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
import importlib.util
import io
import os
import tempfile
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py'))
relay = importlib.util.module_from_spec(spec)
spec.loader.exec_module(relay)
class Clock:
def __init__(self):
self.t = 0.0
def __call__(self):
return self.t
class RelayTests(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
open(self.path, 'w').close()
self.out = io.StringIO()
self.clock = Clock()
def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3):
return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock)
def append(self, text):
with open(self.path, 'a') as fh:
fh.write(text)
def test_history_is_not_replayed_and_new_lines_are_prefixed(self):
self.append('old line from a previous container\n')
r = self.make()
r.poll_once()
self.assertEqual(self.out.getvalue(), '')
self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
r.poll_once()
self.assertEqual(self.out.getvalue(),
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
def test_partial_line_is_held_until_complete(self):
r = self.make()
r.poll_once()
self.append('Thread 1 is abo')
r.poll_once()
self.assertEqual(self.out.getvalue(), '')
self.append('ut to kill the process.\n')
r.poll_once()
self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue())
def test_control_characters_cannot_forge_lines(self):
r = self.make()
r.poll_once()
self.append('a\x1b[31mb\r\x00c\n')
r.poll_once()
self.assertEqual(self.out.getvalue().count('\n'), 1)
self.assertNotIn('\x1b', self.out.getvalue())
def test_rate_limit_passes_a_full_dump_then_summarises(self):
r = self.make(burst=200, per_min=60)
r.poll_once()
self.append(''.join('dump line %d\n' % i for i in range(150)))
r.poll_once()
self.assertEqual(self.out.getvalue().count('\n'), 150)
self.out.truncate(0), self.out.seek(0)
self.append(''.join('flap %d\n' % i for i in range(500)))
r.poll_once()
self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left
self.clock.t += 600 # bucket refills
self.out.truncate(0), self.out.seek(0)
self.append('after storm\n')
r.poll_once()
r.tick()
self.assertIn('[relay] suppressed ', self.out.getvalue())
self.assertIn('after storm', self.out.getvalue())
def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self):
r = self.make(max_bytes=100, backups=2)
r.poll_once()
self.append('x' * 150 + '\n')
r.poll_once()
r.rotate_if_needed()
self.assertEqual(os.path.getsize(self.path), 0)
self.assertTrue(os.path.exists(self.path + '.1'))
self.append('after rotate\n')
r.poll_once()
self.assertIn('after rotate', self.out.getvalue())
def test_missing_file_is_not_an_error(self):
os.remove(self.path)
r = self.make()
r.poll_once()
self.append('late\n')
r.poll_once()
self.assertIn('late', self.out.getvalue())
# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic ---------------
def reload_block(pid, n_proxies=100):
"""What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config)."""
L = ['[NOTICE] (%d) : Reloading HAProxy' % pid,
'[NOTICE] (%d) : Initializing new worker (%d)' % (pid, pid + 100),
'[NOTICE] (%d) : haproxy version is 3.2.26-1~bpo13+1' % (pid + 100),
'[NOTICE] (%d) : path to executable is /usr/sbin/haproxy' % (pid + 100)]
for i in range(6):
L.append("[WARNING] (%d) : config : parsing [/etc/haproxy/haproxy.cfg:%d] : a 'http-request' rule "
"placed after a 'use_backend' rule will still be processed before." % (pid + 100, 800 + i))
L.append("[WARNING] (%d) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid)
L.append('[NOTICE] (%d) : Loading success.' % pid)
for i in range(n_proxies):
L.append('[WARNING] (%d) : Proxy be_%d stopped (cumulated conns: FE: 0, BE: 12).' % (pid + 50, i))
L.append('[WARNING] (%d) : Former worker (%d) exited with code 0 (Exit)' % (pid, pid + 50))
return L
DUMP = [
'Thread 1 is about to kill the process (pid 6631).',
'* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0',
' call trace(15):',
' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0',
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
' lock_hist: R:TASK_WQ U:TASK_WQ',
' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
'',
"Hint: when reporting this bug to developers, please check if a core file was",
" produced, open it with 'gdb', issue 't a a bt full', check that the",
'[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.',
'Please check that you are running an up to date and maintained version of haproxy and open a bug report.',
'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/',
'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html',
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
'[WARNING] (6328) : All workers exited. Exiting... (134)',
]
KEYS = ('about to kill', 'Thread 1 :', 'Thread 2 :', 'Thread 3 :', 'Thread 4 :', 'Current worker',
'exited with code 134', 'exit-on-failure', 'All workers exited', 'call trace', 'ha_dump_backtrace',
'bug in haproxy', 'Please check that you are running', 'produced, open it with')
def scenario(mod, burst=400, drained=True):
"""4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts."""
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = mod.Relay('/nonexistent', out, burst, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
lines = []
for k in range(4):
lines += reload_block(100 * (k + 1))
n_before = len(lines)
lines += DUMP
for l in lines:
if l.strip(): # poll_once drops blank lines before _emit
r._emit(l)
t[0] += 0.003
r.tick()
return {'out': out.getvalue(), 'err': err.getvalue(), 'reload_lines': n_before, 'bucket': r.tokens, 'r': r}
class CapturedScenarioTests(unittest.TestCase):
def test_drained_bucket_then_panic_forwards_every_crash_line(self):
f = scenario(relay)
self.assertGreater(f['reload_lines'], 400) # would drain a 400-line bucket
e = f['err']
for k in KEYS:
self.assertIn(k, e, k)
# the whole dump, line for line (blank lines are dropped by design)
got = [l[len(relay.PREFIX):] for l in e.split('\n') if l]
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
def test_dump_is_forwarded_in_full_even_with_an_empty_bucket(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for l in ('benign %d' % i for i in range(10)):
r._emit(l)
self.assertEqual(out.getvalue(), '')
for l in DUMP:
if l.strip():
r._emit(l)
self.assertEqual(err.getvalue().count('\n'), len([d for d in DUMP if d.strip()]))
self.assertEqual(r.suppressed, 10)
def test_reload_noise_is_collapsed_not_drained(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for k in range(10):
for l in reload_block(100 * (k + 1)):
r._emit(l)
self.assertEqual(r.tokens, 400.0) # bucket untouched by 1000+ noise lines
self.assertEqual(r.suppressed, 0)
self.assertEqual(err.getvalue(), '')
summ = [l for l in out.getvalue().split('\n') if 'reload/start ok' in l]
self.assertEqual(len(summ), 10) # exactly one per reload
self.assertNotIn('Proxy be_', out.getvalue())
self.assertNotIn('parsing [', out.getvalue())
self.assertLess(out.getvalue().count('\n'), 15)
def test_suppressed_lines_are_summarised_per_minute(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(105):
r._emit('flapping server state %d' % i)
self.assertNotIn('[relay] suppressed', out.getvalue())
t[0] += 61
r.tick()
self.assertRegex(out.getvalue(), r'\[relay\] suppressed 100 lines')
self.assertEqual(r.suppressed, 0)
self.assertEqual(err.getvalue(), '')
def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr(self):
# docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "<N>" is NOT parsed
f = scenario(relay)
self.assertIn('[relay] reload/start ok', f['out'])
self.assertNotIn('Thread', f['out'])
self.assertNotIn('Current worker', f['out'])
self.assertNotIn('[ALERT]', f['out'])
self.assertNotIn('reload/start ok', f['err'])
for text in (f['out'], f['err']):
for l in text.split('\n'):
self.assertFalse(l.startswith('<') or l.startswith(relay.PREFIX + '<'), l)
def test_clean_worker_exit_is_not_crash_evidence(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
r._emit('[WARNING] (1) : Former worker (5) exited with code 0 (Exit)')
r._emit('[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)')
self.assertEqual(err.getvalue().count('\n'), 1)
self.assertIn('code 143', err.getvalue())
def test_crash_classes_never_limited(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)',
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy',
'[WARNING] (6328) : All workers exited. Exiting... (134)',
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
'* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0']
for l in evidence:
r._emit(l)
self.assertEqual(err.getvalue().count('\n'), len(evidence))
FORGED = [
'GET /FATAL/SIGSEGV/segfault HTTP/1.1',
'[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout',
'[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN',
'[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)',
'Host: Current worker (1) exited with code 134 (Aborted)',
'Host: [ALERT] (1) : forged alert',
'x-forwarded-for: Thread 1 is about to kill the process (pid 1).',
'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1',
'ha_panic watchdog FATAL SIGABRT',
'backend ALERT_Current worker_exited with code 134',
'[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN',
]
def test_forged_keywords_stay_rate_limited(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for l in self.FORGED * 50:
r._emit(l)
self.assertEqual(err.getvalue(), '')
self.assertFalse(r.in_dump)
self.assertEqual(r.suppressed, len(self.FORGED) * 50)
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget
# a forged header in the middle of normal traffic cannot open a block either
r._emit('proxy: Thread 1 is about to kill the process (pid 1).')
r._emit('free text that would be a dump line')
self.assertFalse(r.in_dump)
self.assertEqual(err.getvalue(), '')
def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
r._emit("GET /Loading success HTTP/1.1")
self.assertEqual(r.routine, 0)
self.assertIn('Loading success', out.getvalue())
def test_flood_of_dump_headers_is_capped_by_the_window_budget(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(20000):
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0')
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
n = err.getvalue().count('\n')
self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently
t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again
before = err.getvalue().count('\n')
r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)')
self.assertEqual(err.getvalue().count('\n'), before + 1)
def test_bound_closed_block_cannot_rearm_during_cooldown(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(relay.DUMP_MAX_LINES + 5):
r._emit('never closed %d' % i)
self.assertFalse(r.in_dump)
r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header
self.assertFalse(r.in_dump)
r._emit('call trace line after the second header')
self.assertNotIn('call trace line after', err.getvalue())
t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again
r._emit('Thread 1 is about to kill the process (pid 3).')
self.assertTrue(r.in_dump)
def test_hostile_and_very_long_lines(self):
r = self.make_relay()
r.poll_once()
self.append('A' * 500000 + '\n')
self.append('\x1b]0;evil\x07 [ALERT] forged\x00\r\n')
r.poll_once()
outs = self.out.getvalue().split('\n')
self.assertTrue(all(len(l) < relay.MAX_LINE + 200 for l in outs))
self.assertNotIn('\x1b', self.out.getvalue())
self.assertNotIn('\x00', self.out.getvalue())
self.assertEqual(self.out.getvalue().count('\n'), 2)
def test_unterminated_flood_is_bounded_in_memory(self):
r = self.make_relay()
r.poll_once()
for _ in range(3):
self.append('B' * 400000) # never a newline
r.poll_once()
self.assertLessEqual(len(r.pending), relay.MAX_PENDING)
def test_unclosed_dump_block_cannot_disable_the_limiter(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(relay.DUMP_MAX_LINES + 500):
r._emit('never closed %d' % i)
self.assertFalse(r.in_dump)
self.assertLessEqual(err.getvalue().count('\n'), relay.DUMP_MAX_LINES + 3)
self.assertGreaterEqual(r.suppressed, 400)
def test_dump_block_is_closed_by_time_bound(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
t[0] += relay.DUMP_MAX_SECONDS + 1
r._emit('later benign line')
self.assertFalse(r.in_dump)
self.assertNotIn('later benign', err.getvalue())
# --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence --------
TENANT_LINES = [
"[ALERT] (10) : backend 'be_tenant1' has no server available!",
"[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.",
"[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.",
"[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.",
]
def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit(self.TENANT_LINES[0])
t[0] += 0.001
t[0] += 5.0
for l in DUMP:
if l.strip():
r._emit(l)
got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l]
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
self.assertIn('exited with code 134', err.getvalue())
self.assertNotIn('exited with code 134', out.getvalue())
self.assertNotIn('has no server available', err.getvalue())
def test_tenant_lines_are_their_own_rate_limited_stdout_class(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)])
self.assertEqual(err.getvalue(), '') # never PRIORITY 3
self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2)
self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped
self.assertEqual(r.tokens, 400.0) # general bucket untouched
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched
self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET)
self.assertGreater(r.tenant_suppressed, 1000)
t[0] += relay.SUMMARY_EVERY + 1
r.tick()
self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines')
self.assertEqual(r.tenant_suppressed, 0)
def test_other_alerts_are_not_crash_evidence(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
r._emit('[ALERT] (12) : config : unable to load')
r._emit("[ALERT] (12) : backend 'x' has no server available!")
self.assertEqual(err.getvalue(), '')
self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr
def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
self.assertEqual(r.budget_left, 0)
before = err.getvalue().count('\n')
for l in DUMP:
if l.strip():
r._emit(l)
self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
def test_rotated_backup_is_mode_0640(self):
r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0)
os.chmod(self.path, 0o644)
self.append('x' * 100 + '\n')
r.rotate_if_needed()
self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640)
# shared helpers for the file-driven tests
def setUp(self):
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
open(self.path, 'w').close()
self.out = io.StringIO()
def make_relay(self):
return relay.Relay(self.path, self.out, 1000, 120, 10 ** 6, 3, clock=lambda: 0.0)
def append(self, text):
with open(self.path, 'a') as fh:
fh.write(text)
# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------
def _scenario_ok(mod):
f = scenario(mod)
got = [l[len(mod.PREFIX):] for l in f['err'].split('\n') if l]
crash_ok = all(k in f['err'] for k in KEYS) and got == [d.rstrip() for d in DUMP if d.strip()]
collapse_ok = f['out'].count('reload/start ok') == 4 and 'Proxy be_' not in f['out'] and f['bucket'] > 300
prio_ok = 'Thread' not in f['out'] and 'reload/start ok' not in f['err']
return crash_ok, collapse_ok, prio_ok
class MutantTests(unittest.TestCase):
SRC = os.path.join(HERE, 'haproxy-stderr-relay.py')
def load_mutant(self, old, new):
with open(self.SRC) as fh:
src = fh.read()
self.assertIn(old, src, 'mutation target vanished: update the test')
mutated = src.replace(old, new, 1)
self.assertNotEqual(mutated, src)
d = tempfile.mkdtemp(prefix='hap-relay-mutant-') # private copy, the real file is untouched
path = os.path.join(d, 'mutant.py')
with open(path, 'w') as fh:
fh.write(mutated)
sp = importlib.util.spec_from_file_location('mutant', path)
m = importlib.util.module_from_spec(sp)
sp.loader.exec_module(m)
return m
def test_unmutated_copy_passes_everything(self):
self.assertEqual(_scenario_ok(relay), (True, True, True))
def test_mutant_limiter_applied_to_crash_lines(self):
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:')
crash_ok, _, _ = _scenario_ok(m)
self.assertFalse(crash_ok)
def test_mutant_dump_block_not_tracked(self):
m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:')
crash_ok, _, _ = _scenario_ok(m)
self.assertFalse(crash_ok)
def test_mutant_collapse_disabled(self):
m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:')
_, collapse_ok, _ = _scenario_ok(m)
self.assertFalse(collapse_ok)
def test_mutant_crash_lines_on_stdout(self):
m = self.load_mutant('self._write(self.err, line)', 'self._write(self.out, line)')
_, _, prio_ok = _scenario_ok(m)
self.assertFalse(prio_ok)
def test_mutant_dump_bound_removed(self):
m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ')
err = io.StringIO()
r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(m.DUMP_MAX_LINES if False else 800):
r._emit('never closed %d' % i)
self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block...
def _forged_run(self, mod):
err = io.StringIO()
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for l in CapturedScenarioTests.FORGED * 20:
r._emit(l)
return err.getvalue()
def test_forged_run_clean_on_real_module(self):
self.assertEqual(self._forged_run(relay), '')
def test_mutant_anchors_removed(self):
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):',
"if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):")
self.assertNotEqual(self._forged_run(m), '')
def _flood(self, mod):
err, t = io.StringIO(), [0.0]
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(5000):
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
return err.getvalue().count('\n')
def test_flood_capped_on_real_module(self):
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
def test_mutant_budget_removed(self):
m = self.load_mutant('if self._budget(now, kind):', 'if True:')
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
def _rearm(self, mod):
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO())
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(relay.DUMP_MAX_LINES + 5):
r._emit('x %d' % i)
r._emit('Thread 1 is about to kill the process (pid 2).')
return r.in_dump
def test_mutant_cooldown_removed(self):
self.assertFalse(self._rearm(relay))
m = self.load_mutant(' and now >= self.cooldown_until:', ':')
self.assertTrue(self._rearm(m))
def test_mutant_summary_removed(self):
m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines")
out, t = io.StringIO(), [0.0]
r = m.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=io.StringIO())
for i in range(105):
r._emit('x %d' % i)
t[0] += 61
r.tick()
self.assertNotIn('[relay] suppressed', out.getvalue())
def _tenant_then_panic_ok(self, mod):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!")
t[0] += 0.001
t[0] += 5.0
for l in DUMP:
if l.strip():
r._emit(l)
got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l]
return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue()
def test_tenant_then_panic_clean_on_real_module(self):
self.assertTrue(self._tenant_then_panic_ok(relay))
def test_mutant_wide_alert_exemption_restored(self):
m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker")
self.assertFalse(self._tenant_then_panic_ok(m))
def test_mutant_tenant_class_removed(self):
m = self.load_mutant('if TENANT_RE.match(line):', 'if False:')
self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m))
def _tenant_stdout_ok(self, mod):
out, err = io.StringIO(), io.StringIO()
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
for i in range(1200):
r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout")
return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2
def test_mutant_shared_budget(self):
m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'")
out, err = io.StringIO(), io.StringIO()
r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for i in range(m.CRASH_BUDGET + 500):
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
before = err.getvalue().count('\n')
for l in DUMP:
if l.strip():
r._emit(l)
self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
if __name__ == '__main__':
unittest.main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH)."""
import os
import stat
import subprocess
import tempfile
import time
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh')
PIDOF = """#!/bin/sh
# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds
[ -f "$STATE/started" ] || exit 1
start=$(cat "$STATE/started")
now=$(date +%s.%N)
awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}'
"""
PYTHON = """#!/bin/sh
# stand-in for ensure_haproxy.py: stamp the start, record it
date +%s.%N > "$STATE/started"
date +%s.%N >> "$STATE/starts"
exit ${ENSURE_RC:-0}
"""
class SupervisorTests(unittest.TestCase):
def setUp(self):
self.d = tempfile.mkdtemp(prefix='hap-sup-')
self.state = os.path.join(self.d, 'state')
os.mkdir(self.state)
for name, body in (('pidof', PIDOF), ('python', PYTHON)):
p = os.path.join(self.d, name)
with open(p, 'w') as fh:
fh.write(body)
os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR)
def run_sup(self, seconds, **env):
e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state)
e.setdefault('LIFE', '0')
e.update(env)
try:
subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return 'exited'
except subprocess.TimeoutExpired:
return 'running' # still looping when the clock ran out = alive
def starts(self):
try:
with open(os.path.join(self.state, 'starts')) as fh:
return [float(x) for x in fh.read().split()]
except OSError:
return []
def test_non_integer_interval_does_not_kill_the_supervisor(self):
self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running')
self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying
def test_garbage_interval_and_max_fall_back(self):
self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running')
self.assertGreaterEqual(len(self.starts()), 1)
def test_crash_loop_backs_off_2_5_10_capped(self):
# haproxy dies immediately after every successful start (LIFE=0) -> quick deaths
self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3',
HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8',
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
s = self.starts()
gaps = [round(b - a, 1) for a, b in zip(s, s[1:])]
self.assertGreaterEqual(len(s), 4, gaps)
# gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4
self.assertLess(gaps[0], gaps[1] + 0.3, gaps)
self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps)
self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps)
self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps)
def test_stable_haproxy_resets_the_backoff(self):
# alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth
self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3',
HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8',
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
s = self.starts()
gaps = [b - a for a, b in zip(s, s[1:])]
self.assertGreaterEqual(len(s), 3, gaps)
self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top
def test_running_haproxy_is_left_alone(self):
with open(os.path.join(self.state, 'started'), 'w') as fh:
fh.write(str(time.time()))
self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running')
self.assertEqual(self.starts(), [])
if __name__ == '__main__':
unittest.main()
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""Offline tests for scripts/verify-published-tags.sh (final publish gate).
CI run 5381 failed because buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'`
and prints the whole inspect text, while buildx 0.38 prints a bare digest. A stub
`imagetools` emulates BOTH output styles (plus `--raw`), so no registry or Docker
is needed. Held invariants:
- the 0.28-style (full text) and 0.38-style (bare digest) outputs both verify
- a reported digest that disagrees with the sha256 of the raw manifest fails
- a missing tag, a platform mismatch, or different digests across registries /
tags fail the job
- build-push.yaml's verify step uses the script, and release/* + rc-* tags
resolve to ONLY <version>-rc and <sha> (never latest)
Running: python3 scripts/test-verify-published-tags.py (needs jq)
"""
import hashlib
import json
import os
import re
import stat
import subprocess
import tempfile
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
SCRIPT = os.path.join(HERE, 'verify-published-tags.sh')
WORKFLOW = os.path.join(ROOT, '.gitea', 'workflows', 'build-push.yaml')
STUB = r'''#!/usr/bin/env python3
import hashlib, json, os, sys
a = sys.argv[1:]
mode = os.environ['STUB_MODE'] # v028 | v038
table = json.load(open(os.environ['STUB_TABLE'])) # ref -> {"raw": str, "lie": optional digest}
assert a[0] == 'inspect'
ref = [x for x in a[1:] if not x.startswith('-') and not x.startswith('{{')][0]
ent = table.get(ref)
if ent is None:
sys.stderr.write('ERROR: %s: not found\n' % ref); sys.exit(1)
raw = ent['raw'].encode() + (b'\n' if ent.get('nl') else b'')
digest = ent.get('lie') or 'sha256:' + hashlib.sha256(ent['raw'].encode()).hexdigest()
if '--raw' in a:
sys.stdout.buffer.write(raw); sys.exit(0)
if '--format' in a:
if mode == 'v038':
print(digest)
else: # 0.28: --format ignored, whole text printed
print('Name: ' + ref); print('MediaType: application/vnd.oci.image.index.v1+json')
print('Digest: ' + digest)
sys.exit(0)
print('Name: ' + ref)
print('MediaType: application/vnd.oci.image.index.v1+json')
print('Digest: ' + digest)
print(' ')
print('Manifests: ')
for m in json.loads(ent['raw'])['manifests']:
print(' Name: %s@%s' % (ref, m['digest']))
print(' Digest: sha256:' + 'f' * 64) # indented: must never be mistaken for the index digest
print(' Platform: %s/%s' % (m['platform']['os'], m['platform']['architecture']))
'''
def index(archs, salt=''):
ms = [{'mediaType': 'application/vnd.oci.image.manifest.v1+json',
'digest': 'sha256:' + hashlib.sha256((x + salt).encode()).hexdigest(), 'size': 1,
'platform': {'architecture': x, 'os': 'linux'}} for x in archs]
ms.append({'mediaType': 'application/vnd.oci.image.manifest.v1+json',
'digest': 'sha256:' + '7' * 64, 'size': 1,
'platform': {'architecture': 'unknown', 'os': 'unknown'},
'annotations': {'vnd.docker.reference.type': 'attestation-manifest'}})
return json.dumps({'schemaVersion': 2, 'mediaType': 'application/vnd.oci.image.index.v1+json',
'manifests': ms}, separators=(',', ':'))
REPOS = ['reg.one/a/img', 'reg.two/b/img']
TAGS = ['1.2-rc', 'abc123']
class VerifyTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.stub = os.path.join(self.tmp.name, 'stub')
open(self.stub, 'w').write(STUB)
os.chmod(self.stub, os.stat(self.stub).st_mode | stat.S_IEXEC)
self.table = os.path.join(self.tmp.name, 'table.json')
def tearDown(self):
self.tmp.cleanup()
def table_for(self, raw, **over):
t = {'%s:%s' % (r, g): {'raw': raw} for r in REPOS for g in TAGS}
for k, v in over.items():
ref = k.replace('__', ':').replace('_', '/', 0)
t[ref] = v
return t
def run_script(self, table, mode, platforms='linux/amd64,linux/arm64', tags=TAGS):
json.dump(table, open(self.table, 'w'))
args = [SCRIPT, '--platforms', platforms]
for r in REPOS:
args += ['--repo', r]
for t in tags:
args += ['--tag', t]
env = dict(os.environ, IMAGETOOLS=self.stub, STUB_MODE=mode, STUB_TABLE=self.table)
return subprocess.run(args, env=env, capture_output=True, text=True, timeout=60)
def test_buildx_028_full_text_output_verifies(self):
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v028')
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
self.assertIn('verified:', r.stdout)
self.assertNotIn('bad digest', r.stdout)
def test_buildx_038_bare_digest_output_verifies(self):
r = self.run_script(self.table_for(index(['amd64', 'arm64'])), 'v038')
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
def test_trailing_newline_on_raw_output_tolerated(self):
raw = index(['amd64', 'arm64'])
t = {'%s:%s' % (r, g): {'raw': raw, 'nl': True} for r in REPOS for g in TAGS}
self.assertEqual(self.run_script(t, 'v028').returncode, 0)
def test_amd64_only_publish_matches_platforms(self):
raw = index(['amd64'])
self.assertEqual(self.run_script(self.table_for(raw), 'v028', 'linux/amd64').returncode, 0)
r = self.run_script(self.table_for(raw), 'v028', 'linux/amd64,linux/arm64')
self.assertEqual(r.returncode, 1)
self.assertIn('platform mismatch', r.stdout)
def test_missing_tag_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
del t['reg.two/b/img:abc123']
r = self.run_script(t, 'v038')
self.assertEqual(r.returncode, 1)
self.assertIn('MISSING', r.stdout)
def test_different_digest_across_registries_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
t['reg.two/b/img:1.2-rc'] = {'raw': index(['amd64', 'arm64'], salt='other')}
for mode in ('v028', 'v038'):
r = self.run_script(t, mode)
self.assertEqual(r.returncode, 1, mode)
self.assertIn('digest mismatch', r.stdout)
def test_reported_digest_disagreeing_with_raw_manifest_fails(self):
t = self.table_for(index(['amd64', 'arm64']))
t['reg.one/a/img:1.2-rc']['lie'] = 'sha256:' + '0' * 64
for mode in ('v028', 'v038'):
r = self.run_script(t, mode)
self.assertEqual(r.returncode, 1, mode)
self.assertIn('cross-check FAILED', r.stdout)
def test_garbage_manifest_fails(self):
t = self.table_for('not json')
self.assertEqual(self.run_script(t, 'v038').returncode, 1)
class WorkflowTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
import yaml
cls.src = open(WORKFLOW).read()
cls.doc = yaml.safe_load(cls.src)
cls.steps = cls.doc['jobs']['Build-and-Push']['steps']
def step(self, name):
return [s for s in self.steps if s.get('name') == name][0]
def test_verify_step_uses_script_not_format_digest(self):
run = self.step('Verify published tags in every registry')['run']
self.assertIn('scripts/verify-published-tags.sh', run)
self.assertNotIn('{{.Manifest.Digest}}', run)
def test_run_blocks_are_valid_bash(self):
for s in self.steps:
if 'run' in s:
p = subprocess.run(['bash', '-n'], input=s['run'], text=True, capture_output=True)
self.assertEqual(p.returncode, 0, '%s: %s' % (s.get('name'), p.stderr))
def test_release_and_rc_never_publish_latest(self):
run = self.step('Resolve tags')['run']
m = re.search(r'refs/heads/release/\*\|refs/tags/rc-\*\)\s*\n\s*tags="([^"]*)"', run)
self.assertIsNotNone(m)
self.assertEqual(m.group(1), '${VERSION}-rc ${SHA}')
main = re.search(r'refs/heads/main\)\s*\n\s*tags="([^"]*)"', run)
self.assertIsNotNone(main)
self.assertEqual(main.group(1), 'latest ${VERSION} ${SHA}')
# latest appears in no other tag assignment
self.assertEqual(len(re.findall(r'tags="[^"]*latest', run)), 1)
if __name__ == '__main__':
unittest.main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# Final publish gate: every <repo>:<tag> must resolve, carry exactly the
# platforms the assemble step reported, and have ONE index digest across all
# tags and all registries. Any miss or mismatch exits 1.
#
# verify-published-tags.sh --platforms linux/amd64,linux/arm64 \
# --repo <registry/path> [--repo ...] --tag <tag> [--tag ...]
#
# The index digest is computed in a buildx-VERSION-INDEPENDENT way. CI run 5381
# showed buildx v0.28.0 ignores `--format '{{.Manifest.Digest}}'` and prints the
# whole inspect text, while 0.38 prints a bare digest. So the digest comes from:
# 1. sha256 of the raw manifest bytes (`inspect --raw`) - always, authoritative
# (a content digest IS the sha256 of the manifest bytes);
# 2. the strictly anchored `Digest:` line of the default text output, if present;
# 3. the `--format '{{.Manifest.Digest}}'` output, ONLY if it is a bare digest
# (0.28's full-text output is ignored here, not an error).
# Every source that is available must agree, else the ref fails.
# Platforms are read from the raw JSON (also version independent).
#
# Env: IMAGETOOLS overrides the command (default "docker buildx imagetools");
# the CI step sets it to "$BUILDX imagetools"; the test suite uses a stub.
set -uo pipefail
IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools}
platforms=""; repos=(); tags=()
while [ $# -gt 0 ]; do
case "$1" in
--platforms) platforms=${2-}; shift 2 ;;
--repo) repos+=("${2-}"); shift 2 ;;
--tag) tags+=("${2-}"); shift 2 ;;
*) echo "verify-published-tags: unknown argument: $1" >&2; exit 2 ;;
esac
done
[ -n "$platforms" ] && [ ${#repos[@]} -gt 0 ] && [ ${#tags[@]} -gt 0 ] \
|| { echo "::error::verify-published-tags: --platforms, --repo and --tag are required"; exit 2; }
command -v jq >/dev/null || { echo "::error::verify-published-tags: jq is required"; exit 2; }
want=$(printf '%s' "$platforms" | tr ',' '\n' | sort | tr '\n' ' ')
work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
digest_re='^sha256:[0-9a-f]{64}$'
ref_digest=""; ref_name=""; fail=0
for r in "${repos[@]}"; do
for t in "${tags[@]}"; do
ref="${r}:${t}"
raw="$work/raw"
if ! timeout 90 $IMAGETOOLS inspect --raw "$ref" >"$raw" 2>"$work/err" || [ ! -s "$raw" ]; then
echo "::error::MISSING or unreadable: ${ref}: $(head -c 300 "$work/err")"; fail=1; continue
fi
# Raw bytes as served, and with trailing newlines stripped (some buildx
# versions append one when printing); the content digest is one of the two.
h1="sha256:$(sha256sum <"$raw" | cut -d' ' -f1)"
h2="sha256:$(printf '%s' "$(cat "$raw")" | sha256sum | cut -d' ' -f1)"
# Source 2: strictly anchored top-level "Digest:" line of the text output.
textd=""
if timeout 90 $IMAGETOOLS inspect "$ref" >"$work/text" 2>/dev/null; then
mapfile -t lines < <(grep -E '^Digest:[[:space:]]+sha256:[0-9a-f]{64}[[:space:]]*$' "$work/text" | sed -E 's/^Digest:[[:space:]]+//; s/[[:space:]]+$//')
[ ${#lines[@]} -eq 1 ] && textd=${lines[0]}
fi
# Source 3: bare --format output, only when it really is a bare digest.
fmtd=""
if out=$(timeout 90 $IMAGETOOLS inspect "$ref" --format '{{.Manifest.Digest}}' 2>/dev/null); then
out=$(printf '%s' "$out" | tr -d '[:space:]')
[[ "$out" =~ $digest_re ]] && fmtd=$out
fi
d=""
for cand in "$textd" "$fmtd"; do
[ -n "$cand" ] || continue
if [ "$cand" != "$h1" ] && [ "$cand" != "$h2" ]; then
echo "::error::digest cross-check FAILED for ${ref}: reported ${cand}, raw manifest hashes to ${h1}"; fail=1; d="!"; break
fi
d=$cand
done
[ "$d" = "!" ] && continue
[ -n "$d" ] || d=$h2
[[ "$d" =~ $digest_re ]] || { echo "::error::bad digest for ${ref}: ${d}"; fail=1; continue; }
if [ -z "$textd$fmtd" ]; then
echo "::warning::no reported digest for ${ref}; using sha256 of the raw manifest"
fi
if ! got=$(jq -r '.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"' "$raw" 2>/dev/null | sort | tr '\n' ' '); then
echo "::error::unparseable manifest for ${ref}"; fail=1; continue
fi
if [ "$got" != "$want" ]; then
echo "::error::platform mismatch for ${ref}: have [${got}] want [${want}]"; fail=1
fi
echo "ok ${ref} ${d} [${got}]"
if [ -z "$ref_digest" ]; then ref_digest="$d"; ref_name="$ref"
elif [ "$d" != "$ref_digest" ]; then
echo "::error::digest mismatch: ${ref}=${d} vs ${ref_name}=${ref_digest}"; fail=1
fi
done
done
[ "$fail" = 0 ] || { echo "::error::tag verification FAILED"; exit 1; }
echo "verified: ${tags[*]} in ${#repos[@]} registries, digest ${ref_digest}, platforms ${platforms}"