fix(relay): never rate-limit crash evidence; collapse reload noise; stdout/stderr priority split

The 3.2 canary showed the relay's 400-line bucket, drained by ~110 reload lines
per reload, dropping a crash dump (journal kept only 'Thread 1 is about to kill').
Crash lines and whole dump blocks now bypass the bucket (bounded block), routine
reload noise is collapsed to one line per reload, the remainder keeps the limit
with a per-minute '[relay] suppressed N lines' summary. Benign lines go to stdout
(journald PRIORITY 6), crash evidence to stderr (PRIORITY 3).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
shadowdaoandClaude Sonnet 5.5 committed 2026-11-16 15:04:14 -08:00
1 parent a753d300db
commit 5e27870608
3 files changed
+418 -17

No files matched your search

+141 -15
View File
@@ -17,9 +17,27 @@ Properties
* Rare by nature: HAProxy only writes here on start/reload/server state
changes/crashes. The per-request access log is NOT here (it goes to the
receiver socket), so this does not feed the WHP AI monitor a firehose.
* Rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST lines burst,
HAPROXY_STDERR_RELAY_PER_MIN refill): a full thread dump (~150 lines) passes,
a flapping-health-check storm is summarised ("N lines suppressed").
* Crash evidence is NEVER rate-limited and NEVER summarised away: lines that
match CRASH_RE (ha_panic, "Thread N is about to kill", "Current worker ...
exited with code N", [ALERT], watchdog, "Thread N : id=", FATAL, SIGSEGV,
SIGABRT, ...) and every line of a dump block (from a panic/watchdog header to
the master's "All workers exited" line, bounded by
DUMP_MAX_LINES and DUMP_MAX_SECONDS so a never-closed block cannot disable
the limiter) are forwarded in full even when the bucket is empty. They go to
STDERR.
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
"Initializing new worker", "Loading success", ...) never touches the bucket:
it is counted and collapsed into ONE "[relay] reload/start ok: N routine
lines collapsed" line per reload (plus one on the minute tick for the late
"Proxy stopped" lines). The full text stays in the stderr file.
* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST
lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute
"[relay] suppressed N lines" summary.
* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and
stderr -> PRIORITY 3 (err) and does NOT parse "<N>" prefixes. So benign
lines (routine, rate-limited remainder, relay summaries) are written to
STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority.
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
Truncating is safe because HAProxy's fd is O_APPEND.
@@ -32,11 +50,38 @@ Environment
HAPROXY_STDERR_RELAY_POLL (0.5 s)
"""
import os
import re
import shutil
import sys
import time
PREFIX = 'haproxy-stderr: '
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
DUMP_MAX_LINES = 3000 # a dump block never bypasses the limiter longer than this
DUMP_MAX_SECONDS = 30.0
SUMMARY_EVERY = 60.0
# Crash evidence: forwarded unconditionally (to stderr). Deliberately broad.
CRASH_RE = re.compile(
r'ha_panic|is about to kill|Current worker|exited with code (?!0\b)|\[ALERT\]'
r'|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=|\bFATAL\b|segfault|SIGSEGV|SIGABRT'
r'|unexpectedly died|bug in haproxy|exit-on-failure|All workers exited|^Hint: when reporting this bug',
re.I)
# A dump block starts at one of these and runs until DUMP_END_RE (or a bound).
DUMP_START_RE = re.compile(
r'ha_panic|is about to kill|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=', re.I)
DUMP_END_RE = re.compile(r'All workers exited')
# A new start/reload cycle also closes a dump block.
CYCLE_RE = re.compile(r'Reloading HAProxy|Initializing new worker|Loading success')
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
ROUTINE_RE = re.compile(
r'Proxy \S+ stopped \(cumulated conns'
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
r"|Former worker \(\d+\) exited with code 0\b"
r'|Reloading HAProxy|Initializing new worker|Loading success'
r'|path to executable is|haproxy version is'
r"|Can't load stats-file|'option forwardfor' ignored")
def _env_num(name, default, cast=int):
@@ -48,16 +93,24 @@ def _env_num(name, default, cast=int):
class Relay:
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic):
self.path, self.out = path, out
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None):
self.path = path
self.out = out # benign lines (docker journald: PRIORITY 6)
self.err = err if err is not None else out # crash lines (PRIORITY 3)
self.burst, self.per_min = burst, per_min
self.max_bytes, self.backups = max_bytes, backups
self.clock = clock
self.tokens = float(burst)
self.last = clock()
self.suppressed = 0
self.routine = 0 # collapsed routine lines since the last summary
self.summary_at = clock()
self.in_dump = False
self.dump_lines = 0
self.dump_since = 0.0
self.offset = None # None = first pass: start at EOF (do not replay history)
self.pending = ''
self._last_stream = None
def _take(self):
now = self.clock()
@@ -68,14 +121,77 @@ class Relay:
return True
return False
def _write(self, stream, text):
# keep the two streams in emission order: flush the other one first
if self._last_stream is not None and self._last_stream is not stream:
self._last_stream.flush()
stream.write(PREFIX + text + '\n')
self._last_stream = stream
def _flush_all(self):
for st in {id(self.out): self.out, id(self.err): self.err}.values():
try:
st.flush()
except Exception:
pass
def _summaries(self, force=False):
"""Per-minute summary of what the limiter / collapse withheld."""
now = self.clock()
if not force and now - self.summary_at < SUMMARY_EVERY:
return
self.summary_at = now
if self.routine:
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
if self.suppressed:
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
% self.suppressed)
self.suppressed = 0
def _end_dump(self, why=None):
self.in_dump = False
if why:
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
def _emit(self, line):
now = self.clock()
if self.in_dump:
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
self._end_dump('bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
elif CYCLE_RE.search(line) and not CRASH_RE.search(line):
self._end_dump()
if CRASH_RE.search(line) or self.in_dump:
# crash evidence: bypasses the bucket, the collapse and the summary
if not self.in_dump and DUMP_START_RE.search(line):
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
if self.in_dump:
self.dump_lines += 1
self._write(self.err, line)
if self.in_dump and DUMP_END_RE.search(line):
self._end_dump()
return
if ROUTINE_RE.search(line):
self.routine += 1
if 'Loading success' in line: # one summary per start/reload
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
self._summaries()
return
if self._take():
if self.suppressed:
self.out.write(PREFIX + '[relay] %d line(s) suppressed by rate limit\n' % self.suppressed)
self.suppressed = 0
self.out.write(PREFIX + line + '\n')
self._write(self.out, line)
else:
self.suppressed += 1
self._summaries()
@staticmethod
def _clean(line):
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
if len(line) > MAX_LINE:
line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line)
return line
def rotate_if_needed(self):
try:
@@ -113,28 +229,38 @@ class Relay:
text = self.pending + chunk.decode('utf-8', 'replace')
lines = text.split('\n')
self.pending = lines.pop() # incomplete trailing line stays buffered
if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line
lines.append(self.pending)
self.pending = ''
for line in lines:
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
line = self._clean(line)
if line:
self._emit(line)
self.out.flush()
self._summaries()
self._flush_all()
def tick(self):
"""Idle-time housekeeping: emit the per-minute summaries even when no line arrives."""
self._summaries()
self._flush_all()
def main():
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
relay = Relay(path, sys.stderr,
relay = Relay(path, sys.stdout,
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
)
err=sys.stderr)
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
sys.stderr.write('haproxy-stderr-relay: following %s\n' % path)
sys.stderr.flush()
sys.stdout.write('haproxy-stderr-relay: following %s\n' % path)
sys.stdout.flush()
ticks = 0
while True:
try:
relay.poll_once()
relay.tick()
ticks += 1
if ticks % 120 == 0:
relay.rotate_if_needed()