diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index fecb4bf..e71c7b8 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -1,9 +1,15 @@ name: HAProxy Manager Build and Push -run-name: ${{ gitea.actor }} pushed a change to main +run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }} on: push: branches: - main + # Release candidates: a push to release/* builds the SAME image but publishes + # ONLY :-rc and :, never :latest and never the plain : + # (see the "Resolve tags" step). Main keeps its behaviour. + - 'release/**' + tags: + - 'rc-*' # Manual publish (e.g. after a [skip ci] merge, or to re-publish after a # failed arm64 leg). workflow_dispatch: @@ -77,6 +83,30 @@ jobs: id: ver run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT" + # Publishing strategy. main: :latest + : + : (unchanged). + # release/* branch or rc-* tag: :-rc + : ONLY. A release candidate + # is deployed per host by explicit tag (durable per-host pin in WHP); :latest is + # moved to it by a later main publish only after the staged rollout succeeded. + # Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused + # rather than guessed at. + - name: Resolve tags + id: tags + env: + REF: ${{ gitea.ref }} + VERSION: ${{ steps.ver.outputs.version }} + SHA: ${{ gitea.sha }} + run: | + case "$REF" in + refs/heads/main) + tags="latest ${VERSION} ${SHA}" ;; + refs/heads/release/*|refs/tags/rc-*) + tags="${VERSION}-rc ${SHA}" ;; + *) + echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;; + esac + echo "tags=${tags}" >> "$GITHUB_OUTPUT" + echo "publishing tags: ${tags}" + # amd64: required. Pushed untagged by digest to both registries (same # digest in each); tags are only created by the assemble step below. - name: Build amd64 (push by digest) @@ -103,19 +133,22 @@ jobs: PKG_CACHE=${{ steps.pkgcache.outputs.url }} outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true - # Three tags per registry: :latest (moving), : (human-readable + # main: three tags per registry: :latest (moving), : (human-readable # release), : (immutable, guaranteed-unique rollback target). + # release/* or rc-*: :-rc and : only (see "Resolve tags"). - name: Assemble tag indexes env: AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} ARM64_OUTCOME: ${{ steps.arm64.outcome }} VERSION: ${{ steps.ver.outputs.version }} - SHA: ${{ gitea.sha }} + TAGS: ${{ steps.tags.outputs.tags }} run: | + tag_args=() + for t in $TAGS; do tag_args+=(--tag "$t"); done scripts/assemble-multiarch-index.sh \ --amd64 "$AMD64_DIGEST" \ --arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \ --repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \ --repo ghcr.io/shadowdao/haproxy-manager-base \ - --tag latest --tag "$VERSION" --tag "$SHA" + "${tag_args[@]}" diff --git a/Dockerfile b/Dockerfile index f12d533..5b71e6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,19 +41,26 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \ # libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream # has no trixie packages and would need a source build.) # -# The repo only keeps the CURRENT patch of each series, so the pin is the SERIES -# (HAPROXY_UPSTREAM=3.2 -> apt preference `3.2.*`), and the build: -# * FAILS unless the installed package AND `haproxy -v` both report that series -# (a 3.2 -> 3.3/3.4 jump can never arrive by accident: change the ARG); -# * records the exact resolved version in /haproxy/HAPROXY_VERSION and the -# `haproxy -c` gate below proves the new binary accepts the fleet config; -# * for reproducibility pass --build-arg HAPROXY_EXACT=3.2.26 to also pin the -# patch (fails loudly when the repo has moved on - bump deliberately). +# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 .. +# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any +# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian +# package version, HAPROXY_EXACT (default below), and the build: +# * pins apt to it (`Pin: version `; NOTE a bare +# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1", +# and with a non-matching pin apt falls back to Debian's own 3.0.11 package); +# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v` +# reports the series (so an unreachable repo, which would leave Debian's +# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident); +# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c` +# gate below proves the binary accepts the fleet config. +# Taking a new patch is a DELIBERATE edit of the default below (or +# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never +# passes it, so a rebuild is reproducible. # The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat, # fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with # signed-by, never fetched at build time. The fingerprint is asserted below. ARG HAPROXY_UPSTREAM=3.2 -ARG HAPROXY_EXACT= +ARG HAPROXY_EXACT=3.2.26-1~bpo13+1 # OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 # (CT130 pkg-cache, https only; CI sets it only after probing the cache with the # pinned CA). Unset = the upstream Debian repositories, exactly as before. The @@ -70,20 +77,24 @@ COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/hap RUN /usr/local/sbin/pkg-cache-repos.sh apt && \ apt update -y && apt dist-upgrade -y && \ apt install -y --no-install-recommends gnupg ca-certificates && \ - gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg | awk -F: '/^fpr:/{print $10; exit}' | grep -qx 'AEF2348766F371C689A7360095A42FE8353525F9' && \ + keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \ + [ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \ + [ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \ + apt purge -y gnupg && \ echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \ apt update -y && \ - pin="${HAPROXY_EXACT:-${HAPROXY_UPSTREAM}.*}" && \ + pin="${HAPROXY_EXACT}" && \ printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \ apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \ rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \ deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \ + [ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \ case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \ haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \ haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \ if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \ mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \ - echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \ + echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \ /usr/local/sbin/pkg-cache-repos.sh restore && \ apt clean && rm -rf /var/lib/apt/lists/* WORKDIR /haproxy @@ -141,6 +152,7 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \ && python3 /haproxy/scripts/test-default-splash.py \ && python3 /haproxy/scripts/test-access-log-receiver.py \ && python3 /haproxy/scripts/test-haproxy-stderr-relay.py \ + && python3 /haproxy/scripts/test-haproxy-supervisor.py \ && python3 /haproxy/scripts/test-config-rollback.py \ && python3 /haproxy/scripts/test-cert-write-safety.py \ && python3 /haproxy/scripts/test-cert-scripts.py \ diff --git a/haproxy_manager.py b/haproxy_manager.py index 0bdc5ee..7465ce1 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -799,8 +799,8 @@ def _stderr_log_path(): path = HAPROXY_STDERR_LOG try: os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'a'): - pass + # 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable) + os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640)) return path except OSError: return '/tmp/haproxy-stderr.log' @@ -823,7 +823,8 @@ def spawn_haproxy_master(): except OSError: offset = 0 subprocess.run( - ['sh', '-c', 'log=$1; shift; "$@" >"$log" 2>&1 &', 'sh', log_path] + cmd, + ['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; ' + '"$@" >"$log" 2>&1 &', 'sh', log_path] + cmd, check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) diff --git a/scripts/haproxy-stderr-relay.py b/scripts/haproxy-stderr-relay.py index 5f70e16..f5119b9 100755 --- a/scripts/haproxy-stderr-relay.py +++ b/scripts/haproxy-stderr-relay.py @@ -27,6 +27,13 @@ Properties are forwarded in full even when the bucket is empty, within a total budget of CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter and are counted in the suppressed summary). They go to STDERR. +* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!", + "Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints + the first at [ALERT]: every tenant container stop/crash/deploy makes them. + Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own + "[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI + monitor through `docker logs haproxy-manager`; rate-limiting them is intended. + Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend. * Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)", config "parsing [...]" warnings, "Former worker (N) exited with code 0", "Initializing new worker", "Loading success", ...) never touches the bucket: @@ -71,13 +78,24 @@ SUMMARY_EVERY = 60.0 # Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET. CRASH_RE = re.compile( r'^(?:' - r'\[ALERT\]\s+\(\d+\)\s*:' # any HAProxy alert + r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)' r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+' r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died' r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b' r'|Thread \d+ is about to kill the process' r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ ' r')') +# Tenant-driven state lines. WHP gives every tenant server `check`, so every +# tenant container stop/crash/OOM/deploy and every host-wide container restart +# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash +# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so +# they can neither reach priority 3 nor spend the general bucket / crash budgets. +TENANT_RE = re.compile( + r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:' + r"backend '[^']*' has no server available!" + r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)' + r'|Health check for (?:backup )?server \S+ ' + r')') # A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE. DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process') DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b') @@ -96,6 +114,10 @@ LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success # Total exemption budget: even real-looking evidence cannot flood without bound. CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70) CRASH_WINDOW = 600.0 # seconds +DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that + # ordinary evidence lines (worker exits, alerts) can never spend +TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute) +TENANT_PER_MIN = 30 DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long @@ -126,6 +148,11 @@ class Relay: self.cooldown_until = 0.0 self.budget_left = CRASH_BUDGET self.window_start = clock() + self.dump_budget_left = DUMP_BUDGET + self.dump_window_start = clock() + self.tenant_tokens = float(TENANT_BURST) + self.tenant_last = clock() + self.tenant_suppressed = 0 self.offset = None # None = first pass: start at EOF (do not replay history) self.pending = '' self._last_stream = None @@ -139,6 +166,15 @@ class Relay: return True return False + def _take_tenant(self): + now = self.clock() + self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0) + self.tenant_last = now + if self.tenant_tokens >= 1: + self.tenant_tokens -= 1 + return True + return False + def _write(self, stream, text): # keep the two streams in emission order: flush the other one first if self._last_stream is not None and self._last_stream is not stream: @@ -163,13 +199,26 @@ class Relay: self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)' % self.routine) self.routine = 0 + if self.tenant_suppressed: + self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines ' + '(rate limit; full text in the stderr file)' % self.tenant_suppressed) + self.tenant_suppressed = 0 if self.suppressed: self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)' % self.suppressed) self.suppressed = 0 - def _budget(self, now): - """Take one unit of the crash-evidence budget; False once the window's budget is spent.""" + def _budget(self, now, kind='crash'): + """Take one unit of a budget; False once the window's budget is spent. + + kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget).""" + if kind == 'dump': + if now - self.dump_window_start >= CRASH_WINDOW: + self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET + if self.dump_budget_left > 0: + self.dump_budget_left -= 1 + return True + return False if now - self.window_start >= CRASH_WINDOW: self.window_start, self.budget_left = now, CRASH_BUDGET if self.budget_left > 0: @@ -191,7 +240,9 @@ class Relay: elif CYCLE_RE.match(line): self._end_dump(now) if self.in_dump or CRASH_RE.match(line): - if self._budget(now): + is_dump = self.in_dump or bool(DUMP_START_RE.match(line)) + kind = 'dump' if is_dump else 'crash' + if self._budget(now, kind): # crash evidence: bypasses the bucket, the collapse and the summary if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until: self.in_dump, self.dump_lines, self.dump_since = True, 0, now @@ -211,6 +262,13 @@ class Relay: self.routine = 0 self._summaries() return + if TENANT_RE.match(line): + if self._take_tenant(): + self._write(self.out, line) + else: + self.tenant_suppressed += 1 + self._summaries() + return if self._take(): self._write(self.out, line) else: @@ -233,6 +291,7 @@ class Relay: if os.path.exists(src): os.replace(src, '%s.%d' % (self.path, i)) shutil.copyfile(self.path, self.path + '.1') + os.chmod(self.path + '.1', 0o640) with open(self.path, 'r+b') as fh: fh.truncate(0) self.offset = 0 diff --git a/scripts/haproxy-supervisor.sh b/scripts/haproxy-supervisor.sh new file mode 100755 index 0000000..f832942 --- /dev/null +++ b/scripts/haproxy-supervisor.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# In-container haproxy supervisor (started in the background by start-up.sh). +# +# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching +# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134, +# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart +# policy never fires, and haproxy is down until the external host watchdog +# full-restarts the whole container minutes later (dropping every connection). +# This loop revives haproxy in place within one interval. +# +# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports +# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and +# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone. +# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s, +# most of it detection latency + start; the default is now 2 s (was 15). +# +# Two backoffs: +# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its +# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of +# re-rendering/re-validating the same broken config every 2 s; +# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH +# seconds (default 10) of that start (a crash loop): extra wait before the +# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default +# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives +# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death +# after a long run is still revived within one interval. +# +# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5, +# "abc") falls back to the default instead of killing this subshell under set -e. +set -eo pipefail + +_posint() { # _posint VALUE DEFAULT + case "$1" in + ''|*[!0-9]*) echo "$2" ;; + *) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;; + esac +} + +INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)" +MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)" +QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)" +STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)" +STEPS="" +for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do + v="$(_posint "$s" 0)" + [ "$v" -ge 1 ] && STEPS="${STEPS} ${v}" +done +[ -n "${STEPS}" ] || STEPS=" 2 5 10 30" + +# step N (1-based, past the end = last), capped at MAX_BACKOFF +_step() { + local n="$1" i=0 v=0 + for s in ${STEPS}; do + i=$((i + 1)); v="$s" + [ "${i}" -ge "${n}" ] && break + done + [ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}" + echo "${v}" +} + +delay="${INTERVAL}" +last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet +quick=0 # consecutive deaths within QUICK_DEATH of a start +while true; do + sleep "${delay}" + if pidof -q haproxy; then + delay="${INTERVAL}" + if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then + quick=0; last_start=-1 + fi + continue + fi + if [ "${last_start}" -ge 0 ]; then + up=$((SECONDS - last_start)) + if [ "${up}" -lt "${QUICK_DEATH}" ]; then + quick=$((quick + 1)) + wait_s="$(_step "${quick}")" + echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2 + sleep "${wait_s}" + pidof -q haproxy && continue + elif [ "${up}" -ge "${STABLE}" ]; then + quick=0 + fi + fi + if python /haproxy/scripts/ensure_haproxy.py 2>&1; then + delay="${INTERVAL}" + last_start="${SECONDS}" + else + delay=$(( delay * 2 )) + [ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}" + fi +done diff --git a/scripts/start-up.sh b/scripts/start-up.sh index e97a3b3..59792fb 100755 --- a/scripts/start-up.sh +++ b/scripts/start-up.sh @@ -92,40 +92,10 @@ cron & # Phase 1: container init python /haproxy/scripts/init.py -# Phase 1.5: in-container haproxy supervisor. -# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching -# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134, -# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart -# policy never fires, and haproxy is down until the external host watchdog -# full-restarts the whole container minutes later (dropping every connection). -# This loop revives haproxy in place within one interval. -# -# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports -# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and -# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone. -# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s, -# most of it detection latency + start; the default is now 2 s (was 15). -# If a restart attempt FAILS (config HAProxy rejects), the loop backs off -# exponentially to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of -# re-rendering/re-validating the same broken config every 2 s. -HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-2}" -HAPROXY_SUPERVISOR_MAX_BACKOFF="${HAPROXY_SUPERVISOR_MAX_BACKOFF:-30}" -( - delay="${HAPROXY_SUPERVISOR_INTERVAL}" - while true; do - sleep "${delay}" - if pidof -q haproxy; then - delay="${HAPROXY_SUPERVISOR_INTERVAL}" - continue - fi - if python /haproxy/scripts/ensure_haproxy.py 2>&1; then - delay="${HAPROXY_SUPERVISOR_INTERVAL}" - else - delay=$(( delay * 2 )) - [ "${delay}" -gt "${HAPROXY_SUPERVISOR_MAX_BACKOFF}" ] && delay="${HAPROXY_SUPERVISOR_MAX_BACKOFF}" - fi - done -) & +# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh: +# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with +# backoff on failed restarts and on crash loops; knobs validated as integers). +/haproxy/scripts/haproxy-supervisor.sh & # Phase 2: WSGI servers # Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT diff --git a/scripts/test-haproxy-stderr-relay.py b/scripts/test-haproxy-stderr-relay.py index ba7ddfe..214c833 100644 --- a/scripts/test-haproxy-stderr-relay.py +++ b/scripts/test-haproxy-stderr-relay.py @@ -242,7 +242,6 @@ class CapturedScenarioTests(unittest.TestCase): r.tokens = 0.0 evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)', '[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM', - '[ALERT] (12) : config : unable to load', '[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)', '[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy', '[WARNING] (6328) : All workers exited. Exiting... (134)', @@ -298,7 +297,7 @@ class CapturedScenarioTests(unittest.TestCase): r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0') r._emit('[WARNING] (1) : All workers exited. Exiting... (134)') n = err.getvalue().count('\n') - self.assertLessEqual(n, relay.CRASH_BUDGET + 5) + self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again before = err.getvalue().count('\n') @@ -362,6 +361,77 @@ class CapturedScenarioTests(unittest.TestCase): self.assertFalse(r.in_dump) self.assertNotIn('later benign', err.getvalue()) + + # --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence -------- + TENANT_LINES = [ + "[ALERT] (10) : backend 'be_tenant1' has no server available!", + "[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.", + "[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.", + "[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.", + ] + + def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit(self.TENANT_LINES[0]) + t[0] += 0.001 + t[0] += 5.0 + for l in DUMP: + if l.strip(): + r._emit(l) + got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l] + self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()]) + self.assertIn('exited with code 134', err.getvalue()) + self.assertNotIn('exited with code 134', out.getvalue()) + self.assertNotIn('has no server available', err.getvalue()) + + def test_tenant_lines_are_their_own_rate_limited_stdout_class(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)]) + self.assertEqual(err.getvalue(), '') # never PRIORITY 3 + self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2) + self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped + self.assertEqual(r.tokens, 400.0) # general bucket untouched + self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched + self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET) + self.assertGreater(r.tenant_suppressed, 1000) + t[0] += relay.SUMMARY_EVERY + 1 + r.tick() + self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines') + self.assertEqual(r.tenant_suppressed, 0) + + def test_other_alerts_are_not_crash_evidence(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit('[ALERT] (12) : config : unable to load') + r._emit("[ALERT] (12) : backend 'x' has no server available!") + self.assertEqual(err.getvalue(), '') + self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr + + + def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop + r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i) + self.assertEqual(r.budget_left, 0) + before = err.getvalue().count('\n') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()])) + + def test_rotated_backup_is_mode_0640(self): + r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0) + os.chmod(self.path, 0o644) + self.append('x' * 100 + '\n') + r.rotate_if_needed() + self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640) + # shared helpers for the file-driven tests def setUp(self): self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-') @@ -465,11 +535,11 @@ class MutantTests(unittest.TestCase): return err.getvalue().count('\n') def test_flood_capped_on_real_module(self): - self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + 5) + self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) def test_mutant_budget_removed(self): - m = self.load_mutant('if self._budget(now):', 'if True:') - self.assertGreater(self._flood(m), relay.CRASH_BUDGET + 5) + m = self.load_mutant('if self._budget(now, kind):', 'if True:') + self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5) def _rearm(self, mod): r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO()) @@ -496,5 +566,49 @@ class MutantTests(unittest.TestCase): self.assertNotIn('[relay] suppressed', out.getvalue()) + def _tenant_then_panic_ok(self, mod): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(1200): + r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!") + t[0] += 0.001 + t[0] += 5.0 + for l in DUMP: + if l.strip(): + r._emit(l) + got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l] + return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue() + + def test_tenant_then_panic_clean_on_real_module(self): + self.assertTrue(self._tenant_then_panic_ok(relay)) + + def test_mutant_wide_alert_exemption_restored(self): + m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker") + self.assertFalse(self._tenant_then_panic_ok(m)) + + def test_mutant_tenant_class_removed(self): + m = self.load_mutant('if TENANT_RE.match(line):', 'if False:') + self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m)) + + def _tenant_stdout_ok(self, mod): + out, err = io.StringIO(), io.StringIO() + r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + for i in range(1200): + r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout") + return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2 + + def test_mutant_shared_budget(self): + m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'") + out, err = io.StringIO(), io.StringIO() + r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for i in range(m.CRASH_BUDGET + 500): + r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i) + before = err.getvalue().count('\n') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()])) + if __name__ == '__main__': unittest.main() diff --git a/scripts/test-haproxy-supervisor.py b/scripts/test-haproxy-supervisor.py new file mode 100644 index 0000000..777c8df --- /dev/null +++ b/scripts/test-haproxy-supervisor.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH).""" +import os +import stat +import subprocess +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh') + +PIDOF = """#!/bin/sh +# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds +[ -f "$STATE/started" ] || exit 1 +start=$(cat "$STATE/started") +now=$(date +%s.%N) +awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}' +""" +PYTHON = """#!/bin/sh +# stand-in for ensure_haproxy.py: stamp the start, record it +date +%s.%N > "$STATE/started" +date +%s.%N >> "$STATE/starts" +exit ${ENSURE_RC:-0} +""" + + +class SupervisorTests(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp(prefix='hap-sup-') + self.state = os.path.join(self.d, 'state') + os.mkdir(self.state) + for name, body in (('pidof', PIDOF), ('python', PYTHON)): + p = os.path.join(self.d, name) + with open(p, 'w') as fh: + fh.write(body) + os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR) + + def run_sup(self, seconds, **env): + e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state) + e.setdefault('LIFE', '0') + e.update(env) + try: + subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + return 'exited' + except subprocess.TimeoutExpired: + return 'running' # still looping when the clock ran out = alive + + def starts(self): + try: + with open(os.path.join(self.state, 'starts')) as fh: + return [float(x) for x in fh.read().split()] + except OSError: + return [] + + def test_non_integer_interval_does_not_kill_the_supervisor(self): + self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running') + self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying + + def test_garbage_interval_and_max_fall_back(self): + self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running') + self.assertGreaterEqual(len(self.starts()), 1) + + def test_crash_loop_backs_off_2_5_10_capped(self): + # haproxy dies immediately after every successful start (LIFE=0) -> quick deaths + self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3', + HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8', + HAPROXY_SUPERVISOR_MAX_BACKOFF='3') + s = self.starts() + gaps = [round(b - a, 1) for a, b in zip(s, s[1:])] + self.assertGreaterEqual(len(s), 4, gaps) + # gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4 + self.assertLess(gaps[0], gaps[1] + 0.3, gaps) + self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps) + self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps) + self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps) + + def test_stable_haproxy_resets_the_backoff(self): + # alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth + self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3', + HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8', + HAPROXY_SUPERVISOR_MAX_BACKOFF='3') + s = self.starts() + gaps = [b - a for a, b in zip(s, s[1:])] + self.assertGreaterEqual(len(s), 3, gaps) + self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top + + def test_running_haproxy_is_left_alone(self): + with open(os.path.join(self.state, 'started'), 'w') as fh: + fh.write(str(time.time())) + self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running') + self.assertEqual(self.starts(), []) + + +if __name__ == '__main__': + unittest.main()