fix(image): exact haproxy pin + relay tenant-alert class + supervisor backoff + rc publishing

I1: ARG HAPROXY_EXACT=3.2.26-1~bpo13+1 is the apt pin; build fails unless dpkg-query equals it exactly (fail closed when the repo is unreachable); comment corrected.
I2: relay no longer exempts every [ALERT]; tenant-driven 'has no server available' / 'Server ... is DOWN|UP' are their own rate-limited stdout class; dump blocks have their own budget.
Minor: keyring must hold exactly one pub key (gnupg purged), stderr log + rotated copies 0640, supervisor extracted to scripts/haproxy-supervisor.sh (integer-validated knobs, crash-loop backoff 2/5/10/30, reset after 60s).
C1: release/* branches and rc-* tags publish only :<VERSION>-rc and :<sha>, never :latest; main unchanged.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
claudeandClaude Sonnet 5.5 committed 2026-11-16 17:33:27 -08:00
1 parent c2e070d172
commit ea2d4994f1
8 files changed
+440 -62

No files matched your search

+37 -4
View File
@@ -1,9 +1,15 @@
name: HAProxy Manager Build and Push
run-name: ${{ gitea.actor }} pushed a change to main
run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }}
on:
push:
branches:
- main
# Release candidates: a push to release/* builds the SAME image but publishes
# ONLY :<VERSION>-rc and :<sha>, never :latest and never the plain :<VERSION>
# (see the "Resolve tags" step). Main keeps its behaviour.
- 'release/**'
tags:
- 'rc-*'
# Manual publish (e.g. after a [skip ci] merge, or to re-publish after a
# failed arm64 leg).
workflow_dispatch:
@@ -77,6 +83,30 @@ jobs:
id: ver
run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT"
# Publishing strategy. main: :latest + :<version> + :<sha> (unchanged).
# release/* branch or rc-* tag: :<version>-rc + :<sha> ONLY. A release candidate
# is deployed per host by explicit tag (durable per-host pin in WHP); :latest is
# moved to it by a later main publish only after the staged rollout succeeded.
# Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused
# rather than guessed at.
- name: Resolve tags
id: tags
env:
REF: ${{ gitea.ref }}
VERSION: ${{ steps.ver.outputs.version }}
SHA: ${{ gitea.sha }}
run: |
case "$REF" in
refs/heads/main)
tags="latest ${VERSION} ${SHA}" ;;
refs/heads/release/*|refs/tags/rc-*)
tags="${VERSION}-rc ${SHA}" ;;
*)
echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;;
esac
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
echo "publishing tags: ${tags}"
# amd64: required. Pushed untagged by digest to both registries (same
# digest in each); tags are only created by the assemble step below.
- name: Build amd64 (push by digest)
@@ -103,19 +133,22 @@ jobs:
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true
# Three tags per registry: :latest (moving), :<version> (human-readable
# main: three tags per registry: :latest (moving), :<version> (human-readable
# release), :<sha> (immutable, guaranteed-unique rollback target).
# release/* or rc-*: :<version>-rc and :<sha> only (see "Resolve tags").
- name: Assemble tag indexes
env:
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
ARM64_DIGEST: ${{ steps.arm64.outputs.digest }}
ARM64_OUTCOME: ${{ steps.arm64.outcome }}
VERSION: ${{ steps.ver.outputs.version }}
SHA: ${{ gitea.sha }}
TAGS: ${{ steps.tags.outputs.tags }}
run: |
tag_args=()
for t in $TAGS; do tag_args+=(--tag "$t"); done
scripts/assemble-multiarch-index.sh \
--amd64 "$AMD64_DIGEST" \
--arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
--repo ghcr.io/shadowdao/haproxy-manager-base \
--tag latest --tag "$VERSION" --tag "$SHA"
"${tag_args[@]}"
+24 -12
View File
@@ -41,19 +41,26 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \
# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream
# has no trixie packages and would need a source build.)
#
# The repo only keeps the CURRENT patch of each series, so the pin is the SERIES
# (HAPROXY_UPSTREAM=3.2 -> apt preference `3.2.*`), and the build:
# * FAILS unless the installed package AND `haproxy -v` both report that series
# (a 3.2 -> 3.3/3.4 jump can never arrive by accident: change the ARG);
# * records the exact resolved version in /haproxy/HAPROXY_VERSION and the
# `haproxy -c` gate below proves the new binary accepts the fleet config;
# * for reproducibility pass --build-arg HAPROXY_EXACT=3.2.26 to also pin the
# patch (fails loudly when the repo has moved on - bump deliberately).
# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 ..
# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any
# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian
# package version, HAPROXY_EXACT (default below), and the build:
# * pins apt to it (`Pin: version <full Debian version>`; NOTE a bare
# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1",
# and with a non-matching pin apt falls back to Debian's own 3.0.11 package);
# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v`
# reports the series (so an unreachable repo, which would leave Debian's
# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident);
# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c`
# gate below proves the binary accepts the fleet config.
# Taking a new patch is a DELIBERATE edit of the default below (or
# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never
# passes it, so a rebuild is reproducible.
# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat,
# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with
# signed-by, never fetched at build time. The fingerprint is asserted below.
ARG HAPROXY_UPSTREAM=3.2
ARG HAPROXY_EXACT=
ARG HAPROXY_EXACT=3.2.26-1~bpo13+1
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only; CI sets it only after probing the cache with the
# pinned CA). Unset = the upstream Debian repositories, exactly as before. The
@@ -70,20 +77,24 @@ COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/hap
RUN /usr/local/sbin/pkg-cache-repos.sh apt && \
apt update -y && apt dist-upgrade -y && \
apt install -y --no-install-recommends gnupg ca-certificates && \
gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg | awk -F: '/^fpr:/{print $10; exit}' | grep -qx 'AEF2348766F371C689A7360095A42FE8353525F9' && \
keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \
[ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \
[ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \
apt purge -y gnupg && \
echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \
apt update -y && \
pin="${HAPROXY_EXACT:-${HAPROXY_UPSTREAM}.*}" && \
pin="${HAPROXY_EXACT}" && \
printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \
rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \
deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \
[ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \
case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \
haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \
if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \
mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \
echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \
echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \
/usr/local/sbin/pkg-cache-repos.sh restore && \
apt clean && rm -rf /var/lib/apt/lists/*
WORKDIR /haproxy
@@ -141,6 +152,7 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \
&& python3 /haproxy/scripts/test-default-splash.py \
&& python3 /haproxy/scripts/test-access-log-receiver.py \
&& python3 /haproxy/scripts/test-haproxy-stderr-relay.py \
&& python3 /haproxy/scripts/test-haproxy-supervisor.py \
&& python3 /haproxy/scripts/test-config-rollback.py \
&& python3 /haproxy/scripts/test-cert-write-safety.py \
&& python3 /haproxy/scripts/test-cert-scripts.py \
+4 -3
View File
@@ -799,8 +799,8 @@ def _stderr_log_path():
path = HAPROXY_STDERR_LOG
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'a'):
pass
# 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable)
os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640))
return path
except OSError:
return '/tmp/haproxy-stderr.log'
@@ -823,7 +823,8 @@ def spawn_haproxy_master():
except OSError:
offset = 0
subprocess.run(
['sh', '-c', 'log=$1; shift; "$@" </dev/null >>"$log" 2>&1 &', 'sh', log_path] + cmd,
['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; '
'"$@" </dev/null >>"$log" 2>&1 &', 'sh', log_path] + cmd,
check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
+63 -4
View File
@@ -27,6 +27,13 @@ Properties
are forwarded in full even when the bucket is empty, within a total budget of
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
and are counted in the suppressed summary). They go to STDERR.
* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!",
"Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints
the first at [ALERT]: every tenant container stop/crash/deploy makes them.
Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own
"[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI
monitor through `docker logs haproxy-manager`; rate-limiting them is intended.
Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend.
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
"Initializing new worker", "Loading success", ...) never touches the bucket:
@@ -71,13 +78,24 @@ SUMMARY_EVERY = 60.0
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
CRASH_RE = re.compile(
r'^(?:'
r'\[ALERT\]\s+\(\d+\)\s*:' # any HAProxy alert
r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
r'|Thread \d+ is about to kill the process'
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
r')')
# Tenant-driven state lines. WHP gives every tenant server `check`, so every
# tenant container stop/crash/OOM/deploy and every host-wide container restart
# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash
# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so
# they can neither reach priority 3 nor spend the general bucket / crash budgets.
TENANT_RE = re.compile(
r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:'
r"backend '[^']*' has no server available!"
r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)'
r'|Health check for (?:backup )?server \S+ '
r')')
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
@@ -96,6 +114,10 @@ LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success
# Total exemption budget: even real-looking evidence cannot flood without bound.
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
CRASH_WINDOW = 600.0 # seconds
DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that
# ordinary evidence lines (worker exits, alerts) can never spend
TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute)
TENANT_PER_MIN = 30
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
@@ -126,6 +148,11 @@ class Relay:
self.cooldown_until = 0.0
self.budget_left = CRASH_BUDGET
self.window_start = clock()
self.dump_budget_left = DUMP_BUDGET
self.dump_window_start = clock()
self.tenant_tokens = float(TENANT_BURST)
self.tenant_last = clock()
self.tenant_suppressed = 0
self.offset = None # None = first pass: start at EOF (do not replay history)
self.pending = ''
self._last_stream = None
@@ -139,6 +166,15 @@ class Relay:
return True
return False
def _take_tenant(self):
now = self.clock()
self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0)
self.tenant_last = now
if self.tenant_tokens >= 1:
self.tenant_tokens -= 1
return True
return False
def _write(self, stream, text):
# keep the two streams in emission order: flush the other one first
if self._last_stream is not None and self._last_stream is not stream:
@@ -163,13 +199,26 @@ class Relay:
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
if self.tenant_suppressed:
self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines '
'(rate limit; full text in the stderr file)' % self.tenant_suppressed)
self.tenant_suppressed = 0
if self.suppressed:
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
% self.suppressed)
self.suppressed = 0
def _budget(self, now):
"""Take one unit of the crash-evidence budget; False once the window's budget is spent."""
def _budget(self, now, kind='crash'):
"""Take one unit of a budget; False once the window's budget is spent.
kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget)."""
if kind == 'dump':
if now - self.dump_window_start >= CRASH_WINDOW:
self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET
if self.dump_budget_left > 0:
self.dump_budget_left -= 1
return True
return False
if now - self.window_start >= CRASH_WINDOW:
self.window_start, self.budget_left = now, CRASH_BUDGET
if self.budget_left > 0:
@@ -191,7 +240,9 @@ class Relay:
elif CYCLE_RE.match(line):
self._end_dump(now)
if self.in_dump or CRASH_RE.match(line):
if self._budget(now):
is_dump = self.in_dump or bool(DUMP_START_RE.match(line))
kind = 'dump' if is_dump else 'crash'
if self._budget(now, kind):
# crash evidence: bypasses the bucket, the collapse and the summary
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
@@ -211,6 +262,13 @@ class Relay:
self.routine = 0
self._summaries()
return
if TENANT_RE.match(line):
if self._take_tenant():
self._write(self.out, line)
else:
self.tenant_suppressed += 1
self._summaries()
return
if self._take():
self._write(self.out, line)
else:
@@ -233,6 +291,7 @@ class Relay:
if os.path.exists(src):
os.replace(src, '%s.%d' % (self.path, i))
shutil.copyfile(self.path, self.path + '.1')
os.chmod(self.path + '.1', 0o640)
with open(self.path, 'r+b') as fh:
fh.truncate(0)
self.offset = 0
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
# In-container haproxy supervisor (started in the background by start-up.sh).
#
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
# policy never fires, and haproxy is down until the external host watchdog
# full-restarts the whole container minutes later (dropping every connection).
# This loop revives haproxy in place within one interval.
#
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
# most of it detection latency + start; the default is now 2 s (was 15).
#
# Two backoffs:
# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its
# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
# re-rendering/re-validating the same broken config every 2 s;
# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH
# seconds (default 10) of that start (a crash loop): extra wait before the
# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default
# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives
# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death
# after a long run is still revived within one interval.
#
# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5,
# "abc") falls back to the default instead of killing this subshell under set -e.
set -eo pipefail
_posint() { # _posint VALUE DEFAULT
case "$1" in
''|*[!0-9]*) echo "$2" ;;
*) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;;
esac
}
INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)"
MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)"
QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)"
STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)"
STEPS=""
for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do
v="$(_posint "$s" 0)"
[ "$v" -ge 1 ] && STEPS="${STEPS} ${v}"
done
[ -n "${STEPS}" ] || STEPS=" 2 5 10 30"
# step N (1-based, past the end = last), capped at MAX_BACKOFF
_step() {
local n="$1" i=0 v=0
for s in ${STEPS}; do
i=$((i + 1)); v="$s"
[ "${i}" -ge "${n}" ] && break
done
[ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}"
echo "${v}"
}
delay="${INTERVAL}"
last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet
quick=0 # consecutive deaths within QUICK_DEATH of a start
while true; do
sleep "${delay}"
if pidof -q haproxy; then
delay="${INTERVAL}"
if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then
quick=0; last_start=-1
fi
continue
fi
if [ "${last_start}" -ge 0 ]; then
up=$((SECONDS - last_start))
if [ "${up}" -lt "${QUICK_DEATH}" ]; then
quick=$((quick + 1))
wait_s="$(_step "${quick}")"
echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2
sleep "${wait_s}"
pidof -q haproxy && continue
elif [ "${up}" -ge "${STABLE}" ]; then
quick=0
fi
fi
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
delay="${INTERVAL}"
last_start="${SECONDS}"
else
delay=$(( delay * 2 ))
[ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}"
fi
done
+4 -34
View File
@@ -92,40 +92,10 @@ cron &
# Phase 1: container init
python /haproxy/scripts/init.py
# Phase 1.5: in-container haproxy supervisor.
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
# policy never fires, and haproxy is down until the external host watchdog
# full-restarts the whole container minutes later (dropping every connection).
# This loop revives haproxy in place within one interval.
#
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
# most of it detection latency + start; the default is now 2 s (was 15).
# If a restart attempt FAILS (config HAProxy rejects), the loop backs off
# exponentially to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
# re-rendering/re-validating the same broken config every 2 s.
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-2}"
HAPROXY_SUPERVISOR_MAX_BACKOFF="${HAPROXY_SUPERVISOR_MAX_BACKOFF:-30}"
(
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
while true; do
sleep "${delay}"
if pidof -q haproxy; then
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
continue
fi
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
else
delay=$(( delay * 2 ))
[ "${delay}" -gt "${HAPROXY_SUPERVISOR_MAX_BACKOFF}" ] && delay="${HAPROXY_SUPERVISOR_MAX_BACKOFF}"
fi
done
) &
# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh:
# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with
# backoff on failed restarts and on crash loops; knobs validated as integers).
/haproxy/scripts/haproxy-supervisor.sh &
# Phase 2: WSGI servers
# Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT
+119 -5
View File
@@ -242,7 +242,6 @@ class CapturedScenarioTests(unittest.TestCase):
r.tokens = 0.0
evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
'[ALERT] (12) : config : unable to load',
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)',
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy',
'[WARNING] (6328) : All workers exited. Exiting... (134)',
@@ -298,7 +297,7 @@ class CapturedScenarioTests(unittest.TestCase):
r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0')
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
n = err.getvalue().count('\n')
self.assertLessEqual(n, relay.CRASH_BUDGET + 5)
self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently
t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again
before = err.getvalue().count('\n')
@@ -362,6 +361,77 @@ class CapturedScenarioTests(unittest.TestCase):
self.assertFalse(r.in_dump)
self.assertNotIn('later benign', err.getvalue())
# --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence --------
TENANT_LINES = [
"[ALERT] (10) : backend 'be_tenant1' has no server available!",
"[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.",
"[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.",
"[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.",
]
def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit(self.TENANT_LINES[0])
t[0] += 0.001
t[0] += 5.0
for l in DUMP:
if l.strip():
r._emit(l)
got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l]
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
self.assertIn('exited with code 134', err.getvalue())
self.assertNotIn('exited with code 134', out.getvalue())
self.assertNotIn('has no server available', err.getvalue())
def test_tenant_lines_are_their_own_rate_limited_stdout_class(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)])
self.assertEqual(err.getvalue(), '') # never PRIORITY 3
self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2)
self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped
self.assertEqual(r.tokens, 400.0) # general bucket untouched
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched
self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET)
self.assertGreater(r.tenant_suppressed, 1000)
t[0] += relay.SUMMARY_EVERY + 1
r.tick()
self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines')
self.assertEqual(r.tenant_suppressed, 0)
def test_other_alerts_are_not_crash_evidence(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
r._emit('[ALERT] (12) : config : unable to load')
r._emit("[ALERT] (12) : backend 'x' has no server available!")
self.assertEqual(err.getvalue(), '')
self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr
def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
self.assertEqual(r.budget_left, 0)
before = err.getvalue().count('\n')
for l in DUMP:
if l.strip():
r._emit(l)
self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
def test_rotated_backup_is_mode_0640(self):
r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0)
os.chmod(self.path, 0o644)
self.append('x' * 100 + '\n')
r.rotate_if_needed()
self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640)
# shared helpers for the file-driven tests
def setUp(self):
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
@@ -465,11 +535,11 @@ class MutantTests(unittest.TestCase):
return err.getvalue().count('\n')
def test_flood_capped_on_real_module(self):
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + 5)
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
def test_mutant_budget_removed(self):
m = self.load_mutant('if self._budget(now):', 'if True:')
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + 5)
m = self.load_mutant('if self._budget(now, kind):', 'if True:')
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
def _rearm(self, mod):
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO())
@@ -496,5 +566,49 @@ class MutantTests(unittest.TestCase):
self.assertNotIn('[relay] suppressed', out.getvalue())
def _tenant_then_panic_ok(self, mod):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
for i in range(1200):
r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!")
t[0] += 0.001
t[0] += 5.0
for l in DUMP:
if l.strip():
r._emit(l)
got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l]
return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue()
def test_tenant_then_panic_clean_on_real_module(self):
self.assertTrue(self._tenant_then_panic_ok(relay))
def test_mutant_wide_alert_exemption_restored(self):
m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker")
self.assertFalse(self._tenant_then_panic_ok(m))
def test_mutant_tenant_class_removed(self):
m = self.load_mutant('if TENANT_RE.match(line):', 'if False:')
self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m))
def _tenant_stdout_ok(self, mod):
out, err = io.StringIO(), io.StringIO()
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
for i in range(1200):
r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout")
return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2
def test_mutant_shared_budget(self):
m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'")
out, err = io.StringIO(), io.StringIO()
r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for i in range(m.CRASH_BUDGET + 500):
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
before = err.getvalue().count('\n')
for l in DUMP:
if l.strip():
r._emit(l)
self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
if __name__ == '__main__':
unittest.main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH)."""
import os
import stat
import subprocess
import tempfile
import time
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh')
PIDOF = """#!/bin/sh
# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds
[ -f "$STATE/started" ] || exit 1
start=$(cat "$STATE/started")
now=$(date +%s.%N)
awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}'
"""
PYTHON = """#!/bin/sh
# stand-in for ensure_haproxy.py: stamp the start, record it
date +%s.%N > "$STATE/started"
date +%s.%N >> "$STATE/starts"
exit ${ENSURE_RC:-0}
"""
class SupervisorTests(unittest.TestCase):
def setUp(self):
self.d = tempfile.mkdtemp(prefix='hap-sup-')
self.state = os.path.join(self.d, 'state')
os.mkdir(self.state)
for name, body in (('pidof', PIDOF), ('python', PYTHON)):
p = os.path.join(self.d, name)
with open(p, 'w') as fh:
fh.write(body)
os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR)
def run_sup(self, seconds, **env):
e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state)
e.setdefault('LIFE', '0')
e.update(env)
try:
subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return 'exited'
except subprocess.TimeoutExpired:
return 'running' # still looping when the clock ran out = alive
def starts(self):
try:
with open(os.path.join(self.state, 'starts')) as fh:
return [float(x) for x in fh.read().split()]
except OSError:
return []
def test_non_integer_interval_does_not_kill_the_supervisor(self):
self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running')
self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying
def test_garbage_interval_and_max_fall_back(self):
self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running')
self.assertGreaterEqual(len(self.starts()), 1)
def test_crash_loop_backs_off_2_5_10_capped(self):
# haproxy dies immediately after every successful start (LIFE=0) -> quick deaths
self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3',
HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8',
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
s = self.starts()
gaps = [round(b - a, 1) for a, b in zip(s, s[1:])]
self.assertGreaterEqual(len(s), 4, gaps)
# gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4
self.assertLess(gaps[0], gaps[1] + 0.3, gaps)
self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps)
self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps)
self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps)
def test_stable_haproxy_resets_the_backoff(self):
# alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth
self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3',
HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8',
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
s = self.starts()
gaps = [b - a for a, b in zip(s, s[1:])]
self.assertGreaterEqual(len(s), 3, gaps)
self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top
def test_running_haproxy_is_left_alone(self):
with open(os.path.join(self.state, 'started'), 'w') as fh:
fh.write(str(time.time()))
self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running')
self.assertEqual(self.starts(), [])
if __name__ == '__main__':
unittest.main()