feat: HAProxy 3.2 LTS (haproxy.debian.net) + capture HAProxy stderr + 2 s supervisor

DRAFT for review, not merged. Context: HAProxy 3.0.11 (Debian trixie) aborts via
its own watchdog in the DNS resolver (haproxy #3101, fixed in 3.0.13/3.1.11/3.2.6).

* Dockerfile: haproxy from https://haproxy.debian.net trixie-backports-3.2 (committed
  signing key, fingerprint asserted, series pin 3.2.*, optional HAPROXY_EXACT),
  native OpenSSL QUIC asserted (no compat layer), /haproxy/HAPROXY_VERSION.
* start HAProxy as `haproxy -W -db` (spawn_haproxy_master): `daemon` sent the
  master's stderr to /dev/null, losing ALERTs and the ha_panic thread dump.
  stderr -> /var/log/haproxy/haproxy-stderr.log, forwarded to container stderr by
  scripts/haproxy-stderr-relay.py (rate limited, rotated).
* supervisor: pidof probe every 2 s (was python every 15 s), exponential backoff
  on failed restarts.
* tests: relay unit tests; rollback/cert stubs print "Loading success.".

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
Claude committed 2026-11-16 13:02:33 -08:00
1 parent 46ea498db3
commit a753d300db
12 files changed
+470 -51

No files matched your search

+12
View File
@@ -34,6 +34,18 @@ def main():
haproxy_manager.logger.warning(
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
)
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
# before it vanished (the relay forwards the same text; this ties it to the
# supervisor event even if the relay is behind).
try:
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
fh.seek(0, 2)
fh.seek(max(0, fh.tell() - 1500))
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
for line in tail:
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
except OSError:
pass
# start_haproxy() validates the config (and regenerates it if invalid)
# before launching, and swallows its own errors, so it will not raise here.
haproxy_manager.start_haproxy()
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
Why
---
HAProxy's master stderr carries what the access log never will: `[ALERT]`
lines, "Current worker (N) exited with code 134", and the complete
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
haproxy-logs volume, so it survives a container recreate). This process tails the
file and re-emits each line on ITS stderr, prefixed, which is the container's
stderr: `docker logs`, and through the docker logging driver the host journal
and Graylog.
Properties
----------
* Rare by nature: HAProxy only writes here on start/reload/server state
changes/crashes. The per-request access log is NOT here (it goes to the
receiver socket), so this does not feed the WHP AI monitor a firehose.
* Rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST lines burst,
HAPROXY_STDERR_RELAY_PER_MIN refill): a full thread dump (~150 lines) passes,
a flapping-health-check storm is summarised ("N lines suppressed").
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
Truncating is safe because HAProxy's fd is O_APPEND.
* Never blocks or affects HAProxy: it only reads a file.
Environment
-----------
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
HAPROXY_STDERR_RELAY_POLL (0.5 s)
"""
import os
import shutil
import sys
import time
PREFIX = 'haproxy-stderr: '
def _env_num(name, default, cast=int):
try:
v = cast(os.environ.get(name, default))
return v if v > 0 else default
except (TypeError, ValueError):
return default
class Relay:
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic):
self.path, self.out = path, out
self.burst, self.per_min = burst, per_min
self.max_bytes, self.backups = max_bytes, backups
self.clock = clock
self.tokens = float(burst)
self.last = clock()
self.suppressed = 0
self.offset = None # None = first pass: start at EOF (do not replay history)
self.pending = ''
def _take(self):
now = self.clock()
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
self.last = now
if self.tokens >= 1:
self.tokens -= 1
return True
return False
def _emit(self, line):
if self._take():
if self.suppressed:
self.out.write(PREFIX + '[relay] %d line(s) suppressed by rate limit\n' % self.suppressed)
self.suppressed = 0
self.out.write(PREFIX + line + '\n')
else:
self.suppressed += 1
def rotate_if_needed(self):
try:
if os.path.getsize(self.path) <= self.max_bytes:
return
for i in range(self.backups, 1, -1):
src = '%s.%d' % (self.path, i - 1)
if os.path.exists(src):
os.replace(src, '%s.%d' % (self.path, i))
shutil.copyfile(self.path, self.path + '.1')
with open(self.path, 'r+b') as fh:
fh.truncate(0)
self.offset = 0
except OSError as exc:
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
def poll_once(self):
try:
size = os.path.getsize(self.path)
except OSError:
if self.offset is None:
self.offset = 0 # file not created yet: whatever appears is new
return
if self.offset is None:
self.offset = size
return
if size < self.offset: # truncated by someone else
self.offset = 0
if size == self.offset:
return
with open(self.path, 'rb') as fh:
fh.seek(self.offset)
chunk = fh.read(1 << 20)
self.offset += len(chunk)
text = self.pending + chunk.decode('utf-8', 'replace')
lines = text.split('\n')
self.pending = lines.pop() # incomplete trailing line stays buffered
for line in lines:
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
if line:
self._emit(line)
self.out.flush()
def main():
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
relay = Relay(path, sys.stderr,
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
)
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
sys.stderr.write('haproxy-stderr-relay: following %s\n' % path)
sys.stderr.flush()
ticks = 0
while True:
try:
relay.poll_once()
ticks += 1
if ticks % 120 == 0:
relay.rotate_if_needed()
except Exception as exc: # never die: the supervisor loop restarts us anyway
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
time.sleep(poll)
if __name__ == '__main__':
main()
+38 -5
View File
@@ -75,6 +75,20 @@ mkdir -p /var/log/haproxy /var/lib/haproxy
cron &
# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and
# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs,
# "exited with code 134", the watchdog/ha_panic thread dump) is appended to
# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's
# stderr so `docker logs` / the docker logging driver / Graylog get it. Started
# BEFORE init.py so the first start's lines are followed. Restarted within 2 s
# if it ever dies; it only reads a file, so it can never affect HAProxy.
(
while true; do
python /haproxy/scripts/haproxy-stderr-relay.py || true
sleep 2
done
) &
# Phase 1: container init
python /haproxy/scripts/init.py
@@ -84,13 +98,32 @@ python /haproxy/scripts/init.py
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
# policy never fires, and haproxy is down until the external host watchdog
# full-restarts the whole container minutes later (dropping every connection).
# This loop revives haproxy in place within one interval. ensure_haproxy.py is
# idempotent — a cheap no-op whenever haproxy is already running.
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}"
# This loop revives haproxy in place within one interval.
#
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
# most of it detection latency + start; the default is now 2 s (was 15).
# If a restart attempt FAILS (config HAProxy rejects), the loop backs off
# exponentially to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
# re-rendering/re-validating the same broken config every 2 s.
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-2}"
HAPROXY_SUPERVISOR_MAX_BACKOFF="${HAPROXY_SUPERVISOR_MAX_BACKOFF:-30}"
(
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
while true; do
sleep "${HAPROXY_SUPERVISOR_INTERVAL}"
python /haproxy/scripts/ensure_haproxy.py 2>&1 || true
sleep "${delay}"
if pidof -q haproxy; then
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
continue
fi
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
else
delay=$(( delay * 2 ))
[ "${delay}" -gt "${HAPROXY_SUPERVISOR_MAX_BACKOFF}" ] && delay="${HAPROXY_SUPERVISOR_MAX_BACKOFF}"
fi
done
) &
+7 -1
View File
@@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR)
# The module opens /var/log/haproxy-manager.log at import time via
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
# spawn_haproxy_master() appends the master's stderr here.
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
os.environ.setdefault('HAPROXY_START_WAIT', '3')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda fn, *a, **kw: _real_file_handler(
@@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
# haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN}
# reject any unusable file in $TEST_CERTS_DIR
cfg=""
start=0
while [ $# -gt 0 ]; do
case "$1" in -f) cfg="$2"; shift ;; esac
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
shift
done
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
@@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
fi
done
fi
# real `haproxy -W -db` prints this once the config is loaded
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
exit 0
""")
+9 -1
View File
@@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR)
# The module opens /var/log/haproxy-manager.log at import time via
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
# spawn_haproxy_master() appends the master's stderr here (default is the
# haproxy-logs volume path, not writable unprivileged).
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
os.environ.setdefault('HAPROXY_START_WAIT', '3')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda fn, *a, **kw: _real_file_handler(
@@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
cfg=""
start=0
while [ $# -gt 0 ]; do
case "$1" in -f) cfg="$2"; shift ;; esac
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
shift
done
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
exit 1
fi
# A real `haproxy -W -db` prints this once the config is loaded;
# spawn_haproxy_master() waits for it.
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
exit 0
""")
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
import importlib.util
import io
import os
import tempfile
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py'))
relay = importlib.util.module_from_spec(spec)
spec.loader.exec_module(relay)
class Clock:
def __init__(self):
self.t = 0.0
def __call__(self):
return self.t
class RelayTests(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
open(self.path, 'w').close()
self.out = io.StringIO()
self.clock = Clock()
def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3):
return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock)
def append(self, text):
with open(self.path, 'a') as fh:
fh.write(text)
def test_history_is_not_replayed_and_new_lines_are_prefixed(self):
self.append('old line from a previous container\n')
r = self.make()
r.poll_once()
self.assertEqual(self.out.getvalue(), '')
self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
r.poll_once()
self.assertEqual(self.out.getvalue(),
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
def test_partial_line_is_held_until_complete(self):
r = self.make()
r.poll_once()
self.append('Thread 1 is abo')
r.poll_once()
self.assertEqual(self.out.getvalue(), '')
self.append('ut to kill the process.\n')
r.poll_once()
self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue())
def test_control_characters_cannot_forge_lines(self):
r = self.make()
r.poll_once()
self.append('a\x1b[31mb\r\x00c\n')
r.poll_once()
self.assertEqual(self.out.getvalue().count('\n'), 1)
self.assertNotIn('\x1b', self.out.getvalue())
def test_rate_limit_passes_a_full_dump_then_summarises(self):
r = self.make(burst=200, per_min=60)
r.poll_once()
self.append(''.join('dump line %d\n' % i for i in range(150)))
r.poll_once()
self.assertEqual(self.out.getvalue().count('\n'), 150)
self.out.truncate(0), self.out.seek(0)
self.append(''.join('flap %d\n' % i for i in range(500)))
r.poll_once()
self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left
self.clock.t += 600 # bucket refills
self.out.truncate(0), self.out.seek(0)
self.append('after storm\n')
r.poll_once()
self.assertIn('suppressed by rate limit', self.out.getvalue())
self.assertIn('after storm', self.out.getvalue())
def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self):
r = self.make(max_bytes=100, backups=2)
r.poll_once()
self.append('x' * 150 + '\n')
r.poll_once()
r.rotate_if_needed()
self.assertEqual(os.path.getsize(self.path), 0)
self.assertTrue(os.path.exists(self.path + '.1'))
self.append('after rotate\n')
r.poll_once()
self.assertIn('after rotate', self.out.getvalue())
def test_missing_file_is_not_an_error(self):
os.remove(self.path)
r = self.make()
r.poll_once()
self.append('late\n')
r.poll_once()
self.assertIn('late', self.out.getvalue())
if __name__ == '__main__':
unittest.main()