feat: HAProxy 3.2 LTS (haproxy.debian.net) + capture HAProxy stderr + 2 s supervisor
DRAFT for review, not merged. Context: HAProxy 3.0.11 (Debian trixie) aborts via its own watchdog in the DNS resolver (haproxy #3101, fixed in 3.0.13/3.1.11/3.2.6). * Dockerfile: haproxy from https://haproxy.debian.net trixie-backports-3.2 (committed signing key, fingerprint asserted, series pin 3.2.*, optional HAPROXY_EXACT), native OpenSSL QUIC asserted (no compat layer), /haproxy/HAPROXY_VERSION. * start HAProxy as `haproxy -W -db` (spawn_haproxy_master): `daemon` sent the master's stderr to /dev/null, losing ALERTs and the ha_panic thread dump. stderr -> /var/log/haproxy/haproxy-stderr.log, forwarded to container stderr by scripts/haproxy-stderr-relay.py (rate limited, rotated). * supervisor: pidof probe every 2 s (was python every 15 s), exponential backoff on failed restarts. * tests: relay unit tests; rollback/cert stubs print "Loading success.". Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
1 parent
46ea498db3
commit
a753d300db
12 files changed
+470
-51
No files matched your search
@@ -34,6 +34,18 @@ def main():
|
||||
haproxy_manager.logger.warning(
|
||||
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
|
||||
)
|
||||
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
|
||||
# before it vanished (the relay forwards the same text; this ties it to the
|
||||
# supervisor event even if the relay is behind).
|
||||
try:
|
||||
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
|
||||
fh.seek(0, 2)
|
||||
fh.seek(max(0, fh.tell() - 1500))
|
||||
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
|
||||
for line in tail:
|
||||
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
|
||||
except OSError:
|
||||
pass
|
||||
# start_haproxy() validates the config (and regenerates it if invalid)
|
||||
# before launching, and swallows its own errors, so it will not raise here.
|
||||
haproxy_manager.start_haproxy()
|
||||
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
|
||||
|
||||
Why
|
||||
---
|
||||
HAProxy's master stderr carries what the access log never will: `[ALERT]`
|
||||
lines, "Current worker (N) exited with code 134", and the complete
|
||||
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
|
||||
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
|
||||
haproxy-logs volume, so it survives a container recreate). This process tails the
|
||||
file and re-emits each line on ITS stderr, prefixed, which is the container's
|
||||
stderr: `docker logs`, and through the docker logging driver the host journal
|
||||
and Graylog.
|
||||
|
||||
Properties
|
||||
----------
|
||||
* Rare by nature: HAProxy only writes here on start/reload/server state
|
||||
changes/crashes. The per-request access log is NOT here (it goes to the
|
||||
receiver socket), so this does not feed the WHP AI monitor a firehose.
|
||||
* Rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST lines burst,
|
||||
HAPROXY_STDERR_RELAY_PER_MIN refill): a full thread dump (~150 lines) passes,
|
||||
a flapping-health-check storm is summarised ("N lines suppressed").
|
||||
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
|
||||
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
|
||||
Truncating is safe because HAProxy's fd is O_APPEND.
|
||||
* Never blocks or affects HAProxy: it only reads a file.
|
||||
|
||||
Environment
|
||||
-----------
|
||||
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
|
||||
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
|
||||
HAPROXY_STDERR_RELAY_POLL (0.5 s)
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
PREFIX = 'haproxy-stderr: '
|
||||
|
||||
|
||||
def _env_num(name, default, cast=int):
|
||||
try:
|
||||
v = cast(os.environ.get(name, default))
|
||||
return v if v > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic):
|
||||
self.path, self.out = path, out
|
||||
self.burst, self.per_min = burst, per_min
|
||||
self.max_bytes, self.backups = max_bytes, backups
|
||||
self.clock = clock
|
||||
self.tokens = float(burst)
|
||||
self.last = clock()
|
||||
self.suppressed = 0
|
||||
self.offset = None # None = first pass: start at EOF (do not replay history)
|
||||
self.pending = ''
|
||||
|
||||
def _take(self):
|
||||
now = self.clock()
|
||||
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
|
||||
self.last = now
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _emit(self, line):
|
||||
if self._take():
|
||||
if self.suppressed:
|
||||
self.out.write(PREFIX + '[relay] %d line(s) suppressed by rate limit\n' % self.suppressed)
|
||||
self.suppressed = 0
|
||||
self.out.write(PREFIX + line + '\n')
|
||||
else:
|
||||
self.suppressed += 1
|
||||
|
||||
def rotate_if_needed(self):
|
||||
try:
|
||||
if os.path.getsize(self.path) <= self.max_bytes:
|
||||
return
|
||||
for i in range(self.backups, 1, -1):
|
||||
src = '%s.%d' % (self.path, i - 1)
|
||||
if os.path.exists(src):
|
||||
os.replace(src, '%s.%d' % (self.path, i))
|
||||
shutil.copyfile(self.path, self.path + '.1')
|
||||
with open(self.path, 'r+b') as fh:
|
||||
fh.truncate(0)
|
||||
self.offset = 0
|
||||
except OSError as exc:
|
||||
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
|
||||
|
||||
def poll_once(self):
|
||||
try:
|
||||
size = os.path.getsize(self.path)
|
||||
except OSError:
|
||||
if self.offset is None:
|
||||
self.offset = 0 # file not created yet: whatever appears is new
|
||||
return
|
||||
if self.offset is None:
|
||||
self.offset = size
|
||||
return
|
||||
if size < self.offset: # truncated by someone else
|
||||
self.offset = 0
|
||||
if size == self.offset:
|
||||
return
|
||||
with open(self.path, 'rb') as fh:
|
||||
fh.seek(self.offset)
|
||||
chunk = fh.read(1 << 20)
|
||||
self.offset += len(chunk)
|
||||
text = self.pending + chunk.decode('utf-8', 'replace')
|
||||
lines = text.split('\n')
|
||||
self.pending = lines.pop() # incomplete trailing line stays buffered
|
||||
for line in lines:
|
||||
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
|
||||
if line:
|
||||
self._emit(line)
|
||||
self.out.flush()
|
||||
|
||||
|
||||
def main():
|
||||
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
||||
relay = Relay(path, sys.stderr,
|
||||
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
|
||||
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
|
||||
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
|
||||
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
|
||||
)
|
||||
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
|
||||
sys.stderr.write('haproxy-stderr-relay: following %s\n' % path)
|
||||
sys.stderr.flush()
|
||||
ticks = 0
|
||||
while True:
|
||||
try:
|
||||
relay.poll_once()
|
||||
ticks += 1
|
||||
if ticks % 120 == 0:
|
||||
relay.rotate_if_needed()
|
||||
except Exception as exc: # never die: the supervisor loop restarts us anyway
|
||||
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+38
-5
@@ -75,6 +75,20 @@ mkdir -p /var/log/haproxy /var/lib/haproxy
|
||||
|
||||
cron &
|
||||
|
||||
# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and
|
||||
# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs,
|
||||
# "exited with code 134", the watchdog/ha_panic thread dump) is appended to
|
||||
# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's
|
||||
# stderr so `docker logs` / the docker logging driver / Graylog get it. Started
|
||||
# BEFORE init.py so the first start's lines are followed. Restarted within 2 s
|
||||
# if it ever dies; it only reads a file, so it can never affect HAProxy.
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/haproxy-stderr-relay.py || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
# Phase 1: container init
|
||||
python /haproxy/scripts/init.py
|
||||
|
||||
@@ -84,13 +98,32 @@ python /haproxy/scripts/init.py
|
||||
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||||
# policy never fires, and haproxy is down until the external host watchdog
|
||||
# full-restarts the whole container minutes later (dropping every connection).
|
||||
# This loop revives haproxy in place within one interval. ensure_haproxy.py is
|
||||
# idempotent — a cheap no-op whenever haproxy is already running.
|
||||
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}"
|
||||
# This loop revives haproxy in place within one interval.
|
||||
#
|
||||
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
|
||||
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
|
||||
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
|
||||
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
|
||||
# most of it detection latency + start; the default is now 2 s (was 15).
|
||||
# If a restart attempt FAILS (config HAProxy rejects), the loop backs off
|
||||
# exponentially to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
|
||||
# re-rendering/re-validating the same broken config every 2 s.
|
||||
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-2}"
|
||||
HAPROXY_SUPERVISOR_MAX_BACKOFF="${HAPROXY_SUPERVISOR_MAX_BACKOFF:-30}"
|
||||
(
|
||||
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
while true; do
|
||||
sleep "${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
python /haproxy/scripts/ensure_haproxy.py 2>&1 || true
|
||||
sleep "${delay}"
|
||||
if pidof -q haproxy; then
|
||||
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
continue
|
||||
fi
|
||||
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
|
||||
delay="${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
else
|
||||
delay=$(( delay * 2 ))
|
||||
[ "${delay}" -gt "${HAPROXY_SUPERVISOR_MAX_BACKOFF}" ] && delay="${HAPROXY_SUPERVISOR_MAX_BACKOFF}"
|
||||
fi
|
||||
done
|
||||
) &
|
||||
|
||||
|
||||
@@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here.
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN}
|
||||
# reject any unusable file in $TEST_CERTS_DIR
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
@@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# real `haproxy -W -db` prints this once the config is loaded
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
@@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here (default is the
|
||||
# haproxy-logs volume path, not writable unprivileged).
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
|
||||
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
# A real `haproxy -W -db` prints this once the config is loaded;
|
||||
# spawn_haproxy_master() waits for it.
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py'))
|
||||
relay = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(relay)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.t = 0.0
|
||||
|
||||
def __call__(self):
|
||||
return self.t
|
||||
|
||||
|
||||
class RelayTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
|
||||
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
|
||||
open(self.path, 'w').close()
|
||||
self.out = io.StringIO()
|
||||
self.clock = Clock()
|
||||
|
||||
def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3):
|
||||
return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock)
|
||||
|
||||
def append(self, text):
|
||||
with open(self.path, 'a') as fh:
|
||||
fh.write(text)
|
||||
|
||||
def test_history_is_not_replayed_and_new_lines_are_prefixed(self):
|
||||
self.append('old line from a previous container\n')
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(),
|
||||
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
|
||||
def test_partial_line_is_held_until_complete(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('Thread 1 is abo')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('ut to kill the process.\n')
|
||||
r.poll_once()
|
||||
self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue())
|
||||
|
||||
def test_control_characters_cannot_forge_lines(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('a\x1b[31mb\r\x00c\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 1)
|
||||
self.assertNotIn('\x1b', self.out.getvalue())
|
||||
|
||||
def test_rate_limit_passes_a_full_dump_then_summarises(self):
|
||||
r = self.make(burst=200, per_min=60)
|
||||
r.poll_once()
|
||||
self.append(''.join('dump line %d\n' % i for i in range(150)))
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 150)
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append(''.join('flap %d\n' % i for i in range(500)))
|
||||
r.poll_once()
|
||||
self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left
|
||||
self.clock.t += 600 # bucket refills
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append('after storm\n')
|
||||
r.poll_once()
|
||||
self.assertIn('suppressed by rate limit', self.out.getvalue())
|
||||
self.assertIn('after storm', self.out.getvalue())
|
||||
|
||||
def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self):
|
||||
r = self.make(max_bytes=100, backups=2)
|
||||
r.poll_once()
|
||||
self.append('x' * 150 + '\n')
|
||||
r.poll_once()
|
||||
r.rotate_if_needed()
|
||||
self.assertEqual(os.path.getsize(self.path), 0)
|
||||
self.assertTrue(os.path.exists(self.path + '.1'))
|
||||
self.append('after rotate\n')
|
||||
r.poll_once()
|
||||
self.assertIn('after rotate', self.out.getvalue())
|
||||
|
||||
def test_missing_file_is_not_an_error(self):
|
||||
os.remove(self.path)
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('late\n')
|
||||
r.poll_once()
|
||||
self.assertIn('late', self.out.getvalue())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user