From a753d300db033313b2d9c4cbed9e20e87253bfa9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Nov 2026 13:02:33 -0800 Subject: [PATCH] feat: HAProxy 3.2 LTS (haproxy.debian.net) + capture HAProxy stderr + 2 s supervisor DRAFT for review, not merged. Context: HAProxy 3.0.11 (Debian trixie) aborts via its own watchdog in the DNS resolver (haproxy #3101, fixed in 3.0.13/3.1.11/3.2.6). * Dockerfile: haproxy from https://haproxy.debian.net trixie-backports-3.2 (committed signing key, fingerprint asserted, series pin 3.2.*, optional HAPROXY_EXACT), native OpenSSL QUIC asserted (no compat layer), /haproxy/HAPROXY_VERSION. * start HAProxy as `haproxy -W -db` (spawn_haproxy_master): `daemon` sent the master's stderr to /dev/null, losing ALERTs and the ha_panic thread dump. stderr -> /var/log/haproxy/haproxy-stderr.log, forwarded to container stderr by scripts/haproxy-stderr-relay.py (rate limited, rotated). * supervisor: pidof probe every 2 s (was python every 15 s), exponential backoff on failed restarts. * tests: relay unit tests; rollback/cert stubs print "Loading success.". Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- CLAUDE.md | 3 +- Dockerfile | 71 +++++---- README.md | 16 ++ VERSION | 2 +- haproxy_manager.py | 104 +++++++++++-- .../haproxy-archive-keyring.gpg | Bin 0 -> 4079 bytes scripts/ensure_haproxy.py | 12 ++ scripts/haproxy-stderr-relay.py | 147 ++++++++++++++++++ scripts/start-up.sh | 43 ++++- scripts/test-cert-write-safety.py | 8 +- scripts/test-config-rollback.py | 10 +- scripts/test-haproxy-stderr-relay.py | 105 +++++++++++++ 12 files changed, 470 insertions(+), 51 deletions(-) create mode 100644 keys/haproxy-debian-net/haproxy-archive-keyring.gpg create mode 100755 scripts/haproxy-stderr-relay.py create mode 100644 scripts/test-haproxy-stderr-relay.py diff --git a/CLAUDE.md b/CLAUDE.md index dff3ec4..92d9403 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -178,4 +178,5 @@ re-reads it on reload, so a failed runtime command must degrade to - **The :8000 API refuses peers other than 127.0.0.1/::1 and the container's default gateway** (`HAPROXY_MGR_API_ALLOWED_PEERS`, default `127.0.0.1,::1,gateway`; ports via `HAPROXY_MGR_PEER_FILTER_PORTS`, default `8000`). haproxy-manager shares `client-net` with every tenant container; host callers arrive from the gateway via docker-proxy. Only effective with the WHP `-p 127.0.0.1:8000:8000` publish (a 0.0.0.0 publish lets a tenant hairpin through the host and arrive from the gateway). A sidecar that must call :8000 directly needs its IP added - gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat - Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/` -- HAProxy is version 3.0.11 \ No newline at end of file +- HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101) +- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7b03dbd..f12d533 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,49 +23,67 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \ org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.licenses="MIT" -# haproxy is PINNED to a reviewed UPSTREAM version (HAPROXY_UPSTREAM), not to one -# Debian package revision. It was previously unpinned, so the binary could move -# under us at Debian's timing - an upstream release that rejected our config would -# have broken an unrelated commit's build, or worse, shipped an edge that refuses -# to start (see the `haproxy -c` gate below for why that matters: a config HAProxy -# rejects leaves the container Up with 80/443 unbound and /health still 200). +# haproxy comes from the UPSTREAM-maintained apt repo https://haproxy.debian.net +# (Vincent Bernat's builds, the Debian maintainer's own), suite +# `trixie-backports-`, NOT from Debian trixie's archive. # -# History: this used to pin the exact Debian version (3.0.11-1+deb13u3). Debian -# security then shipped 3.0.11-1+deb13u4 (same upstream 3.0.11, same arch set) and -# the exact pin silently froze the image on the OLDER, unpatched revision, and -# would hard-fail the build as soon as u3 left the archive. Debian security -# revisions of the SAME upstream version are exactly what we want to take, so: -# * an apt preference (Pin: version 3.0.11-*) makes apt install the NEWEST -# Debian revision of that upstream version on any architecture; -# * the build then FAILS unless the installed package AND `haproxy -v` both -# report that upstream version, so a major/minor bump (3.0 -> 3.1, 3.0.12) -# can never arrive by accident: that still needs this ARG changed and the -# `haproxy -c` gate below to pass. -# The pin and the gate compose: pinned = the version moves deliberately; the gate -# answers immediately whether the new binary still accepts fleet config. +# WHY (2026-11-16 forensics): Debian trixie ships 3.0.11 (+ security revisions). +# That build aborts via HAProxy's own watchdog inside the DNS resolver +# (resolv_process_responses / snr_resolution_cb, haproxy issue #3101) - whole +# edge down 15-52 s per abort on sdbees/whp01. The fix ("dns: bind the +# nameserver sockets to the initiating thread", "resolvers: make +# process_resolvers() single-threaded") is in 3.0.13+, 3.1.11+ and 3.2.6+; +# Debian trixie will not take it. See docs/go-port/haproxy-edge-upgrade-and-logging.md +# in the WHP repo for the version evidence and the rollout plan. # -# To move to another upstream version: change HAPROXY_UPSTREAM, rebuild, let the -# gate verify. Check what Debian offers with: apt-cache madison haproxy -ARG HAPROXY_UPSTREAM=3.0.11 +# Series: 3.2 (LTS, supported to 2030-Q2). Its trixie package is built with +# USE_QUIC=1 against the system OpenSSL 3.5.x NATIVE QUIC API (depends on +# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream +# has no trixie packages and would need a source build.) +# +# The repo only keeps the CURRENT patch of each series, so the pin is the SERIES +# (HAPROXY_UPSTREAM=3.2 -> apt preference `3.2.*`), and the build: +# * FAILS unless the installed package AND `haproxy -v` both report that series +# (a 3.2 -> 3.3/3.4 jump can never arrive by accident: change the ARG); +# * records the exact resolved version in /haproxy/HAPROXY_VERSION and the +# `haproxy -c` gate below proves the new binary accepts the fleet config; +# * for reproducibility pass --build-arg HAPROXY_EXACT=3.2.26 to also pin the +# patch (fails loudly when the repo has moved on - bump deliberately). +# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat, +# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with +# signed-by, never fetched at build time. The fingerprint is asserted below. +ARG HAPROXY_UPSTREAM=3.2 +ARG HAPROXY_EXACT= # OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 # (CT130 pkg-cache, https only; CI sets it only after probing the cache with the # pinned CA). Unset = the upstream Debian repositories, exactly as before. The # apt sources are restored before this RUN ends (restore FAILS the build if any # reference to the cache is left), so no shipped layer carries the cache address. +# NOTE: CT130 does not proxy haproxy.debian.net (see the design doc); that one +# repo is always fetched directly. # Canonical script + docs: cloud-apache-container (scripts/pkg-cache-repos.sh, # docs/pkg-cache.md there); this repo carries byte-identical copies. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ +COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/haproxy-archive-keyring.gpg RUN /usr/local/sbin/pkg-cache-repos.sh apt && \ apt update -y && apt dist-upgrade -y && \ - printf 'Package: haproxy\nPin: version %s-*\nPin-Priority: 1001\n' "${HAPROXY_UPSTREAM}" > /etc/apt/preferences.d/haproxy-upstream-pin && \ + apt install -y --no-install-recommends gnupg ca-certificates && \ + gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg | awk -F: '/^fpr:/{print $10; exit}' | grep -qx 'AEF2348766F371C689A7360095A42FE8353525F9' && \ + echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \ + apt update -y && \ + pin="${HAPROXY_EXACT:-${HAPROXY_UPSTREAM}.*}" && \ + printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \ apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \ rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \ deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \ - case "${deb_ver}" in "${HAPROXY_UPSTREAM}"-*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not upstream ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \ - haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}-" && \ - echo "haproxy pin OK: upstream ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \ + case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \ + haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \ + haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \ + if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \ + mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \ + echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver}" && \ /usr/local/sbin/pkg-cache-repos.sh restore && \ apt clean && rm -rf /var/lib/apt/lists/* WORKDIR /haproxy @@ -122,6 +140,7 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \ && python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \ && python3 /haproxy/scripts/test-default-splash.py \ && python3 /haproxy/scripts/test-access-log-receiver.py \ + && python3 /haproxy/scripts/test-haproxy-stderr-relay.py \ && python3 /haproxy/scripts/test-config-rollback.py \ && python3 /haproxy/scripts/test-cert-write-safety.py \ && python3 /haproxy/scripts/test-cert-scripts.py \ diff --git a/README.md b/README.md index 1feb108..b5d53b1 100644 --- a/README.md +++ b/README.md @@ -492,6 +492,22 @@ Rollout: image bump + recreate with the volume and sysctl (WHP's are in the image, outside the `/etc/haproxy` named volume, so nothing can be shadowed; `haproxy.cfg` is regenerated on start with the new `log` line. +## HAProxy stderr (panic dumps, [ALERT]s) - since 2026.11.4 + +HAProxy is started as `haproxy -W -db` so the master keeps a real stderr (with +the `daemon` keyword HAProxy points it at /dev/null, which discarded the +`[ALERT]` lines, `Current worker (N) exited with code 134` and the whole +watchdog/`ha_panic` thread dump). It is appended to +`/var/log/haproxy/haproxy-stderr.log` (the `haproxy-logs` volume, so it +survives a recreate) and `scripts/haproxy-stderr-relay.py` re-emits every line +on the container's stderr prefixed `haproxy-stderr: ` (rate-limited, file +rotated at 20 MiB x 3). `docker logs haproxy-manager` and whatever docker +logging driver the host configured therefore carry the crash text; the +per-request access log stays out of it. The in-container supervisor probes +with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` seconds (default 2, was 15) and +backs off to `HAPROXY_SUPERVISOR_MAX_BACKOFF` (30) only when a restart attempt +fails. Measured (3.2.26, empty config): crash to listeners back = 1.2-2.0 s. + ## Default Backend Configuration With the default splash on (above), unrouted hosts no longer reach this page; it still serves the blocked-IP and suspended pages, and serves unrouted hosts again when `HAPROXY_DEFAULT_SPLASH=off`. diff --git a/VERSION b/VERSION index 99b89ef..e931320 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.11.3 +2026.11.4 diff --git a/haproxy_manager.py b/haproxy_manager.py index 395c25d..0bdc5ee 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -766,6 +766,91 @@ def is_process_running(process_name): return True return False +# --------------------------------------------------------------------------- +# Launching the HAProxy master (2026-11 forensics change) +# +# haproxy.cfg carries `daemon`, and in daemon mode HAProxy re-points the +# master's stdin/stdout/stderr at /dev/null after it has parsed the config +# (MEASURED on 3.0.11: /proc//fd/2 -> /dev/null). That discarded every +# [ALERT], the "exited with code 134" master line and the whole ha_panic()/ +# watchdog thread dump of the 2026-11 aborts: only the core file survived. +# +# `-db` overrides `daemon` (MEASURED: foreground master, still master-worker, +# reload via the CLI keeps the same master pid), so the master keeps a real +# stderr. We append it to HAPROXY_STDERR_LOG (haproxy-logs volume, survives a +# container recreate) and scripts/haproxy-stderr-relay.py forwards it to the +# container's stderr (-> docker logging driver -> host journald/Graylog). +# +# The master is started as `sh -c '... &'` so it is re-parented to PID 1 exactly +# as the old daemon fork did (PID 1 reaps it; a Popen child would sit as a +# zombie in the API worker and is_process_running('haproxy') would keep +# reporting it alive). +# --------------------------------------------------------------------------- +HAPROXY_STDERR_LOG = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log') +try: + HAPROXY_START_WAIT = float(os.environ.get('HAPROXY_START_WAIT', '10')) +except ValueError: + HAPROXY_START_WAIT = 10.0 + + +def _stderr_log_path(): + """HAPROXY_STDERR_LOG if its directory is writable, else a /tmp fallback + (a missing volume must never stop HAProxy from starting).""" + path = HAPROXY_STDERR_LOG + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'a'): + pass + return path + except OSError: + return '/tmp/haproxy-stderr.log' + + +def spawn_haproxy_master(): + """Start `haproxy -W -db` detached, stderr appended to the stderr log. + + Returns a subprocess.CompletedProcess (returncode 0, .stderr = the new log + text) once HAProxy has loaded its configuration; raises + subprocess.CalledProcessError (.stderr = what HAProxy printed) if it exits + or never comes up within HAPROXY_START_WAIT seconds. Same contract the old + blocking `subprocess.run([... '-W' ...], check=True)` call sites relied on. + """ + cmd = ['haproxy', '-W', '-db', '-S', '/tmp/haproxy-cli,level,admin', + '-f', HAPROXY_CONFIG_PATH] + log_path = _stderr_log_path() + try: + offset = os.path.getsize(log_path) + except OSError: + offset = 0 + subprocess.run( + ['sh', '-c', 'log=$1; shift; "$@" >"$log" 2>&1 &', 'sh', log_path] + cmd, + check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + def new_text(): + try: + with open(log_path, 'rb') as fh: + fh.seek(offset) + return fh.read().decode('utf-8', 'replace') + except OSError: + return '' + + deadline = time.monotonic() + HAPROXY_START_WAIT + started = time.monotonic() + while time.monotonic() < deadline: + text = new_text() + if 'Loading success.' in text: + return subprocess.CompletedProcess(cmd, 0, '', text) + if time.monotonic() - started > 1.0 and not is_process_running('haproxy'): + break + time.sleep(0.1) + text = new_text() + if is_process_running('haproxy') and '[ALERT]' not in text: + # Slow to log (big config) but alive: treat as started. + return subprocess.CompletedProcess(cmd, 0, '', text) + raise subprocess.CalledProcessError(1, cmd, output='', stderr=text[-4000:]) + + # Initialize template engine template_loader = jinja2.FileSystemLoader(TEMPLATE_DIR) template_env = jinja2.Environment(loader=template_loader) @@ -871,12 +956,7 @@ def reload_haproxy(): return jsonify({'status': 'success'}), 200 else: # Start HAProxy if it's not running - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, - capture_output=True, - text=True - ) + result = spawn_haproxy_master() if result.returncode == 0: print("HAProxy started successfully") log_operation('start_haproxy', True) @@ -3778,10 +3858,7 @@ def reload_haproxy_safely(backup_status=None): else: # HAProxy not running, start it try: - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, capture_output=True, text=True - ) + result = spawn_haproxy_master() logger.info("HAProxy started successfully") # Now - and only now - is this config known good. promote_current_config_to_backup() @@ -4114,12 +4191,7 @@ def start_haproxy(): return # Configuration is valid, start HAProxy - result = subprocess.run( - ['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH], - check=True, - capture_output=True, - text=True - ) + result = spawn_haproxy_master() if result.returncode == 0: logger.info("HAProxy started successfully") log_operation('start_haproxy', True, 'HAProxy started successfully') diff --git a/keys/haproxy-debian-net/haproxy-archive-keyring.gpg b/keys/haproxy-debian-net/haproxy-archive-keyring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..17b3fe371dd37d01aadd8104bfedc76a306950ef GIT binary patch literal 4079 zcmbW(XEYq@zQ=K9^g1?zj5d02i4xI`=%V*tf{+oTL|qK-PF1R+|KXdx0cgdj@v z2*cMUbp}S9AToLwU4vgqN6_Vt@F~{k{d4nYa|TU`b@E_gt?(+w`>`!oV6$ z%`fQ6THoZk0gIjG2;B+t5I^>eu!R$2$~Ifl9e9pgUwc776S0kWyTDXP5)a(6kj0O> zZqgT_xS$b>rniq*@v^X$fjX@pAzMQ@4kaIJnrcQgO}6aVhN+(!u;p89k>agL`3MXBe-*0N65W(0r79!Ub!pnA2&Uoj^Rabu~>&a4VPAByOjwr;xu z+~{n39YV6VR$P78BKT+VVA55ax8d$t&*9#{@c2zwQ7BS1sZM$8u^Cku5M577@l`EG zS+F#Tnq5=7x50#rSaOy~p52IIoTG<0#H2837^l6m06J;b@?R-qFT7Z}u=R{!El!({ zq^Ny5kVsM);@US#uee(EEgN=rgz7hbQG~`nB%mWCAZ+3^_k_8)!TdSZ+~6=Le@+GG zzg`vZfJcvl#ax1v5&-f<^n_#tJOD6|n39}~6iCcSK~6?Vd^4JhBjT82qr`kTOWlVlalC_eBeY!}U(5TPa=D}EbZy0t=SNH2~TTYRf zZR^1+13slD99x(Ujvp6}5YMeCB^t_Maj@?iSOHj@B`ym)|Z81y5J@`Y(7^p0ltJ*daT)>l(o zx1s>kkrH2aL@3ZyBUzi7h{_|-AlJ`>7PwcVYHYimef#p-kP}VQkf|x*PXH07=y-1pW*D} z-WQ{i9D42Z$1Hbiv%8_h_Ar^*<9;h#wC+JUT@V-Fzyqi2$^7PYuf&Xj*423ajy9pm zK*F-t#W`E!**9Tx-tNcT%LeV%sC@-cZFT>r^y4IBhmzgoDLWd9UsdfiRP>&=GxZhs z;okD4Q)KIAOG{nvVq!=~m#_3_NK~A78aXZfp*cAmM_WcWc?CK3O44D&yom3HE%cN2 zZzA8U>JH+3)HYmt9`QC|XwHT_4T8%%07I9O!;xeJ+@^iNDB)&7Aw|RuiaP8nX$9`y z68eM@xI=DeGs|+aGh-$Xllmp+1M_&k|HL#RYEg+A=G_-Wtrw9)jM&()Z%Z0|P}&0h z4Rd(G@?}^kfjQ>*T>~!=8x-#+c-~ZpQ#j4V{odpEEk;+SXNEC)4e~1LFfhV&%^Jo& z>(g!03l+{MGjs)(dUr_~FksCnQzfQTG_#(sYSMMZ5)6P0v5YJck5$?!t=PYaeqopu z&0HG#-7Zh0c*ILb2;z7mG-0LZ7&j0Aq76+2O3zFcC?Gzl<7yCWfT;`D6;pAm$8+%x zW{xA+Kh#zSIlk#*qIyc-ULMI=AQiQfIjZeWoq>sbGdlCdFQ=Q)#@=MCr#wu*+$Kf} zF4|cPu$`R_Qr)8aJ4n-_yobb<4R&)2L!TU}DlPCM)a>Omxajf-$b;bbD5bvim#p@# z3d1)>T$|Dy@-KD{vEs*uk=w;U5Zvh5O(AX{HBkw?$6CuvF-UT8 z6>SEaw;0R(OjCBpF=@#E-o~r|uV5okW~@(r=ZZU$&Q`JyF_yIz=*aOpz$H0T*5j@LIY45lJ6kD8vn z^{t8VY?q#)QHxBD2WCQS{k4^?Lil+) zZ8v@r>~JW*a!j1_eYjqs0kw1d7G;nKYU0h?c09m*=zG@|NdwL00C;+jpF#>!a(Jeo z9HSo4ZRU5)d-gyZm3g_wmy{9s^bl`Zvez`IJOI8NhI&&(A&vOqZsU5GZWB&b%E3o> zZ6dPH<%i?FFHI4K zrpUrcb6EyJ*0lYJD~mkKWEGO5z84L)pdU{yi*3Xe^_};qGi7M;kjo9|t8Une%C?I< zkAGP|bV2BdBD9P2(XF@afXY3YoYf#HaRucc80~uL&ZVsy(_J_vqNSKcQTP;QwwQF~G*7vG6~H?e9r*R;9&Tm$yuF^vdwj zGsj)NTNU%q*%lHx;KBR1B$>MsJ zaa!ijyMEv_Nqb4zGXnmpcum-D80JX9u1GAlaa!2%Ou*W;Z6eL-hLSyQVanY{P<``> z1-}u!{fS{1Itv$nz4>Cmhe+nT?r2p|X5O5-a1?X5x@P|;sNQ!q${Xoz>vRW|`^rqS z8HSEYe`rAxxyAAXOzbQsjPz8u-s+7}GrSW%D67ck2=Phim%5F=Jp0X^f@*S)2M!{h zWw`Pw%sF~a@oJ}Xq-TTaTKg;nW#+VxG}{m_$oPU2)80z@n?0m{Y|W`l6NMym=3TU{ z&;@;^D5I~g+xK1$G~9(`cCFJ}MW0|T#$0Ew;)Z~ZAxjJ(R zq2iyac7lbH#2ft_17(MhKSsB%C4pI@Gn|T-+#)hDD?)KDdsTPNk0$I??kZR0>Lpb0 zYrbl^sw9>s#iJ$nG;;+iw0s)0NmG8QwsHpP-*FHZ?*m?1XZ z0Q)qZbb2#3FQInT-gDl)UDG~XUJ{^`Uf6Lt{*zfCFRVm3ny>V=#Hh8PXH|{<2)AiX zTHIdtXme$8O1|a*kxK~~8xp`@(B%+tA6b;s>HRM6e0=$VyI~hh$K(}DpP7&tfRVai zLf9@waEbvRxuxIU7^GP*Zuz{`+UB(O!W3;2h~NUukN$uS)**k#U^Njw42|>E({?+K zlv&9Kr=tYaVtMwiU8K_SjQ2*6HFk>tVfwpYCuRk0+$^^F2z}z+YqhgkGIp_as2-4~$Sg8lmKw!j2?^fUB}LlUFQgS*NQIS|$Im!o9pOUoLo_VgoHbLzt4 z!KLzFg_?S#2PZGrOLb3Mtkw^t7n7e58<)kET(XXJEh%;yhEOq2dw{R?0nSuptTNy9 zQPl9;{kk)Mbe~Vw{(!feF4T(55z+r zc~MY?%O3P6{VXLsSLwJdP`m~p;WpPhyx6f881%LJYo8|o6o{DrT;F^EFx9_pjB|Zl z9x*xVJzUSS1)jkQ*7=n zOITB$m601*)7H0ZB~EZz+wS!x92m$n?g3=+d%rKvGN7FSTjlC{EIl25-EkP({k@$WaQTPUz@}8GUME1FF{)bl*?iWD;ALe%ylI*$0AbZhiv_*+m<$G+(RT%ZA KlN51N<^KT#@qvE; literal 0 HcmV?d00001 diff --git a/scripts/ensure_haproxy.py b/scripts/ensure_haproxy.py index a91755e..ff3107d 100644 --- a/scripts/ensure_haproxy.py +++ b/scripts/ensure_haproxy.py @@ -34,6 +34,18 @@ def main(): haproxy_manager.logger.warning( "[haproxy-supervisor] haproxy process not found — attempting in-place restart" ) + # Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said + # before it vanished (the relay forwards the same text; this ties it to the + # supervisor event even if the relay is behind). + try: + with open(haproxy_manager._stderr_log_path(), 'rb') as fh: + fh.seek(0, 2) + fh.seek(max(0, fh.tell() - 1500)) + tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:] + for line in tail: + haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line) + except OSError: + pass # start_haproxy() validates the config (and regenerates it if invalid) # before launching, and swallows its own errors, so it will not raise here. haproxy_manager.start_haproxy() diff --git a/scripts/haproxy-stderr-relay.py b/scripts/haproxy-stderr-relay.py new file mode 100755 index 0000000..ab5c9a5 --- /dev/null +++ b/scripts/haproxy-stderr-relay.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Forward HAProxy's own stderr to the container's stderr, and bound the file. + +Why +--- +HAProxy's master stderr carries what the access log never will: `[ALERT]` +lines, "Current worker (N) exited with code 134", and the complete +ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master() +launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the +haproxy-logs volume, so it survives a container recreate). This process tails the +file and re-emits each line on ITS stderr, prefixed, which is the container's +stderr: `docker logs`, and through the docker logging driver the host journal +and Graylog. + +Properties +---------- +* Rare by nature: HAProxy only writes here on start/reload/server state + changes/crashes. The per-request access log is NOT here (it goes to the + receiver socket), so this does not feed the WHP AI monitor a firehose. +* Rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST lines burst, + HAPROXY_STDERR_RELAY_PER_MIN refill): a full thread dump (~150 lines) passes, + a flapping-health-check storm is summarised ("N lines suppressed"). +* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES + (default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3). + Truncating is safe because HAProxy's fd is O_APPEND. +* Never blocks or affects HAProxy: it only reads a file. + +Environment +----------- + HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120), + HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3), + HAPROXY_STDERR_RELAY_POLL (0.5 s) +""" +import os +import shutil +import sys +import time + +PREFIX = 'haproxy-stderr: ' + + +def _env_num(name, default, cast=int): + try: + v = cast(os.environ.get(name, default)) + return v if v > 0 else default + except (TypeError, ValueError): + return default + + +class Relay: + def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic): + self.path, self.out = path, out + self.burst, self.per_min = burst, per_min + self.max_bytes, self.backups = max_bytes, backups + self.clock = clock + self.tokens = float(burst) + self.last = clock() + self.suppressed = 0 + self.offset = None # None = first pass: start at EOF (do not replay history) + self.pending = '' + + def _take(self): + now = self.clock() + self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0) + self.last = now + if self.tokens >= 1: + self.tokens -= 1 + return True + return False + + def _emit(self, line): + if self._take(): + if self.suppressed: + self.out.write(PREFIX + '[relay] %d line(s) suppressed by rate limit\n' % self.suppressed) + self.suppressed = 0 + self.out.write(PREFIX + line + '\n') + else: + self.suppressed += 1 + + def rotate_if_needed(self): + try: + if os.path.getsize(self.path) <= self.max_bytes: + return + for i in range(self.backups, 1, -1): + src = '%s.%d' % (self.path, i - 1) + if os.path.exists(src): + os.replace(src, '%s.%d' % (self.path, i)) + shutil.copyfile(self.path, self.path + '.1') + with open(self.path, 'r+b') as fh: + fh.truncate(0) + self.offset = 0 + except OSError as exc: + sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc) + + def poll_once(self): + try: + size = os.path.getsize(self.path) + except OSError: + if self.offset is None: + self.offset = 0 # file not created yet: whatever appears is new + return + if self.offset is None: + self.offset = size + return + if size < self.offset: # truncated by someone else + self.offset = 0 + if size == self.offset: + return + with open(self.path, 'rb') as fh: + fh.seek(self.offset) + chunk = fh.read(1 << 20) + self.offset += len(chunk) + text = self.pending + chunk.decode('utf-8', 'replace') + lines = text.split('\n') + self.pending = lines.pop() # incomplete trailing line stays buffered + for line in lines: + line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip() + if line: + self._emit(line) + self.out.flush() + + +def main(): + path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log') + relay = Relay(path, sys.stderr, + _env_num('HAPROXY_STDERR_RELAY_BURST', 400), + _env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120), + _env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024), + _env_num('HAPROXY_STDERR_LOG_BACKUPS', 3), + ) + poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float) + sys.stderr.write('haproxy-stderr-relay: following %s\n' % path) + sys.stderr.flush() + ticks = 0 + while True: + try: + relay.poll_once() + ticks += 1 + if ticks % 120 == 0: + relay.rotate_if_needed() + except Exception as exc: # never die: the supervisor loop restarts us anyway + sys.stderr.write('haproxy-stderr-relay: %r\n' % exc) + time.sleep(poll) + + +if __name__ == '__main__': + main() diff --git a/scripts/start-up.sh b/scripts/start-up.sh index ba0c403..e97a3b3 100755 --- a/scripts/start-up.sh +++ b/scripts/start-up.sh @@ -75,6 +75,20 @@ mkdir -p /var/log/haproxy /var/lib/haproxy cron & +# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and +# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs, +# "exited with code 134", the watchdog/ha_panic thread dump) is appended to +# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's +# stderr so `docker logs` / the docker logging driver / Graylog get it. Started +# BEFORE init.py so the first start's lines are followed. Restarted within 2 s +# if it ever dies; it only reads a file, so it can never affect HAProxy. +( + while true; do + python /haproxy/scripts/haproxy-stderr-relay.py || true + sleep 2 + done +) & + # Phase 1: container init python /haproxy/scripts/init.py @@ -84,13 +98,32 @@ python /haproxy/scripts/init.py # segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart # policy never fires, and haproxy is down until the external host watchdog # full-restarts the whole container minutes later (dropping every connection). -# This loop revives haproxy in place within one interval. ensure_haproxy.py is -# idempotent — a cheap no-op whenever haproxy is already running. -HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}" +# This loop revives haproxy in place within one interval. +# +# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports +# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and +# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone. +# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s, +# most of it detection latency + start; the default is now 2 s (was 15). +# If a restart attempt FAILS (config HAProxy rejects), the loop backs off +# exponentially to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of +# re-rendering/re-validating the same broken config every 2 s. +HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-2}" +HAPROXY_SUPERVISOR_MAX_BACKOFF="${HAPROXY_SUPERVISOR_MAX_BACKOFF:-30}" ( + delay="${HAPROXY_SUPERVISOR_INTERVAL}" while true; do - sleep "${HAPROXY_SUPERVISOR_INTERVAL}" - python /haproxy/scripts/ensure_haproxy.py 2>&1 || true + sleep "${delay}" + if pidof -q haproxy; then + delay="${HAPROXY_SUPERVISOR_INTERVAL}" + continue + fi + if python /haproxy/scripts/ensure_haproxy.py 2>&1; then + delay="${HAPROXY_SUPERVISOR_INTERVAL}" + else + delay=$(( delay * 2 )) + [ "${delay}" -gt "${HAPROXY_SUPERVISOR_MAX_BACKOFF}" ] && delay="${HAPROXY_SUPERVISOR_MAX_BACKOFF}" + fi done ) & diff --git a/scripts/test-cert-write-safety.py b/scripts/test-cert-write-safety.py index c89e742..ba582f9 100755 --- a/scripts/test-cert-write-safety.py +++ b/scripts/test-cert-write-safety.py @@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR) # The module opens /var/log/haproxy-manager.log at import time via # logging.FileHandler. Redirect that one call so the suite runs unprivileged. _LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +# spawn_haproxy_master() appends the master's stderr here. +os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log')) +os.environ.setdefault('HAPROXY_START_WAIT', '3') _real_file_handler = logging.FileHandler logging.FileHandler = ( lambda fn, *a, **kw: _real_file_handler( @@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ # haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN} # reject any unusable file in $TEST_CERTS_DIR cfg="" + start=0 while [ $# -gt 0 ]; do - case "$1" in -f) cfg="$2"; shift ;; esac + case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac shift done if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then @@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ fi done fi + # real `haproxy -W -db` prints this once the config is loaded + [ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2 exit 0 """) diff --git a/scripts/test-config-rollback.py b/scripts/test-config-rollback.py index 1b50725..6bcd730 100755 --- a/scripts/test-config-rollback.py +++ b/scripts/test-config-rollback.py @@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR) # The module opens /var/log/haproxy-manager.log at import time via # logging.FileHandler. Redirect that one call so the suite runs unprivileged. _LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +# spawn_haproxy_master() appends the master's stderr here (default is the +# haproxy-logs volume path, not writable unprivileged). +os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log')) +os.environ.setdefault('HAPROXY_START_WAIT', '3') _real_file_handler = logging.FileHandler logging.FileHandler = ( lambda fn, *a, **kw: _real_file_handler( @@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\ # haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0 # haproxy -W -S ... -f FILE (start) -> same validation, then exit 0 cfg="" + start=0 while [ $# -gt 0 ]; do - case "$1" in -f) cfg="$2"; shift ;; esac + case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac shift done if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2 exit 1 fi + # A real `haproxy -W -db` prints this once the config is loaded; + # spawn_haproxy_master() waits for it. + [ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2 exit 0 """) diff --git a/scripts/test-haproxy-stderr-relay.py b/scripts/test-haproxy-stderr-relay.py new file mode 100644 index 0000000..466e4ac --- /dev/null +++ b/scripts/test-haproxy-stderr-relay.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build).""" +import importlib.util +import io +import os +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py')) +relay = importlib.util.module_from_spec(spec) +spec.loader.exec_module(relay) + + +class Clock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +class RelayTests(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-') + self.path = os.path.join(self.dir, 'haproxy-stderr.log') + open(self.path, 'w').close() + self.out = io.StringIO() + self.clock = Clock() + + def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3): + return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock) + + def append(self, text): + with open(self.path, 'a') as fh: + fh.write(text) + + def test_history_is_not_replayed_and_new_lines_are_prefixed(self): + self.append('old line from a previous container\n') + r = self.make() + r.poll_once() + self.assertEqual(self.out.getvalue(), '') + self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n') + r.poll_once() + self.assertEqual(self.out.getvalue(), + 'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n') + + def test_partial_line_is_held_until_complete(self): + r = self.make() + r.poll_once() + self.append('Thread 1 is abo') + r.poll_once() + self.assertEqual(self.out.getvalue(), '') + self.append('ut to kill the process.\n') + r.poll_once() + self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue()) + + def test_control_characters_cannot_forge_lines(self): + r = self.make() + r.poll_once() + self.append('a\x1b[31mb\r\x00c\n') + r.poll_once() + self.assertEqual(self.out.getvalue().count('\n'), 1) + self.assertNotIn('\x1b', self.out.getvalue()) + + def test_rate_limit_passes_a_full_dump_then_summarises(self): + r = self.make(burst=200, per_min=60) + r.poll_once() + self.append(''.join('dump line %d\n' % i for i in range(150))) + r.poll_once() + self.assertEqual(self.out.getvalue().count('\n'), 150) + self.out.truncate(0), self.out.seek(0) + self.append(''.join('flap %d\n' % i for i in range(500))) + r.poll_once() + self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left + self.clock.t += 600 # bucket refills + self.out.truncate(0), self.out.seek(0) + self.append('after storm\n') + r.poll_once() + self.assertIn('suppressed by rate limit', self.out.getvalue()) + self.assertIn('after storm', self.out.getvalue()) + + def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self): + r = self.make(max_bytes=100, backups=2) + r.poll_once() + self.append('x' * 150 + '\n') + r.poll_once() + r.rotate_if_needed() + self.assertEqual(os.path.getsize(self.path), 0) + self.assertTrue(os.path.exists(self.path + '.1')) + self.append('after rotate\n') + r.poll_once() + self.assertIn('after rotate', self.out.getvalue()) + + def test_missing_file_is_not_an_error(self): + os.remove(self.path) + r = self.make() + r.poll_once() + self.append('late\n') + r.poll_once() + self.assertIn('late', self.out.getvalue()) + + +if __name__ == '__main__': + unittest.main()