fix(relay): anchor crash patterns to HAProxy message structure; crash-evidence budget + dump cooldown

Security review: unanchored keywords let echoed client data bypass the limiter and open
dump blocks; bounds re-armed immediately. Patterns are now anchored, blocks open only on
the 'Thread N is about to kill' header, a 1000-line/10-min budget caps exempt lines
(overflow goes through the normal bucket and the suppressed summary) and a 60 s cooldown
follows a bound-closed block. Tests: forged keywords, flood, cooldown + 3 new mutants.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6
This commit is contained in:
shadowdaoandClaude Sonnet 5.5 committed 2026-11-16 15:08:46 -08:00
1 parent 5e27870608
commit c2e070d172
3 files changed
+200 -49

No files matched your search

+68 -37
View File
@@ -17,14 +17,16 @@ Properties
* Rare by nature: HAProxy only writes here on start/reload/server state
changes/crashes. The per-request access log is NOT here (it goes to the
receiver socket), so this does not feed the WHP AI monitor a firehose.
* Crash evidence is NEVER rate-limited and NEVER summarised away: lines that
match CRASH_RE (ha_panic, "Thread N is about to kill", "Current worker ...
exited with code N", [ALERT], watchdog, "Thread N : id=", FATAL, SIGSEGV,
SIGABRT, ...) and every line of a dump block (from a panic/watchdog header to
* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE
("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is
about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/
server name never matches) and every line of a dump block (from a panic/watchdog header to
the master's "All workers exited" line, bounded by
DUMP_MAX_LINES and DUMP_MAX_SECONDS so a never-closed block cannot disable
the limiter) are forwarded in full even when the bucket is empty. They go to
STDERR.
DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block
can open; blocks open only on the anchored "Thread N is about to kill" header)
are forwarded in full even when the bucket is empty, within a total budget of
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
and are counted in the suppressed summary). They go to STDERR.
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
"Initializing new worker", "Loading success", ...) never touches the bucket:
@@ -58,30 +60,43 @@ import time
PREFIX = 'haproxy-stderr: '
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
DUMP_MAX_LINES = 3000 # a dump block never bypasses the limiter longer than this
DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this
DUMP_MAX_SECONDS = 30.0
SUMMARY_EVERY = 60.0
# Crash evidence: forwarded unconditionally (to stderr). Deliberately broad.
# Every pattern is ANCHORED to HAProxy's own message structure (captured from
# real 3.2 crashes): a keyword inside a request path / Host header / server
# name that HAProxy echoes into a warning can never match, because the line
# prefix "[ALERT] (PID) :" is written by HAProxy itself.
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
CRASH_RE = re.compile(
r'ha_panic|is about to kill|Current worker|exited with code (?!0\b)|\[ALERT\]'
r'|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=|\bFATAL\b|segfault|SIGSEGV|SIGABRT'
r'|unexpectedly died|bug in haproxy|exit-on-failure|All workers exited|^Hint: when reporting this bug',
re.I)
# A dump block starts at one of these and runs until DUMP_END_RE (or a bound).
DUMP_START_RE = re.compile(
r'ha_panic|is about to kill|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=', re.I)
DUMP_END_RE = re.compile(r'All workers exited')
r'^(?:'
r'\[ALERT\]\s+\(\d+\)\s*:' # any HAProxy alert
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
r'|Thread \d+ is about to kill the process'
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
r')')
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
# A new start/reload cycle also closes a dump block.
CYCLE_RE = re.compile(r'Reloading HAProxy|Initializing new worker|Loading success')
CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)')
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
ROUTINE_RE = re.compile(
r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:'
r'Proxy \S+ stopped \(cumulated conns'
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
r"|Former worker \(\d+\) exited with code 0\b"
r'|Former worker \(\d+\) exited with code 0\b'
r'|Reloading HAProxy|Initializing new worker|Loading success'
r'|path to executable is|haproxy version is'
r"|Can't load stats-file|'option forwardfor' ignored")
r"|config: Can't load stats-file|'option forwardfor' ignored)")
LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success')
# Total exemption budget: even real-looking evidence cannot flood without bound.
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
CRASH_WINDOW = 600.0 # seconds
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
def _env_num(name, default, cast=int):
@@ -108,6 +123,9 @@ class Relay:
self.in_dump = False
self.dump_lines = 0
self.dump_since = 0.0
self.cooldown_until = 0.0
self.budget_left = CRASH_BUDGET
self.window_start = clock()
self.offset = None # None = first pass: start at EOF (do not replay history)
self.pending = ''
self._last_stream = None
@@ -150,31 +168,44 @@ class Relay:
% self.suppressed)
self.suppressed = 0
def _end_dump(self, why=None):
def _budget(self, now):
"""Take one unit of the crash-evidence budget; False once the window's budget is spent."""
if now - self.window_start >= CRASH_WINDOW:
self.window_start, self.budget_left = now, CRASH_BUDGET
if self.budget_left > 0:
self.budget_left -= 1
return True
return False
def _end_dump(self, now, why=None):
self.in_dump = False
if why:
if why: # closed by a bound, not by the master's end line: no re-arming for a while
self.cooldown_until = now + DUMP_COOLDOWN
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
def _emit(self, line):
now = self.clock()
if self.in_dump:
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
self._end_dump('bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
elif CYCLE_RE.search(line) and not CRASH_RE.search(line):
self._end_dump()
if CRASH_RE.search(line) or self.in_dump:
# crash evidence: bypasses the bucket, the collapse and the summary
if not self.in_dump and DUMP_START_RE.search(line):
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
if self.in_dump:
self.dump_lines += 1
self._write(self.err, line)
if self.in_dump and DUMP_END_RE.search(line):
self._end_dump()
return
if ROUTINE_RE.search(line):
self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
elif CYCLE_RE.match(line):
self._end_dump(now)
if self.in_dump or CRASH_RE.match(line):
if self._budget(now):
# crash evidence: bypasses the bucket, the collapse and the summary
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
if self.in_dump:
self.dump_lines += 1
self._write(self.err, line)
if self.in_dump and DUMP_END_RE.match(line):
self._end_dump(now)
return
if self.in_dump: # budget spent mid-block: the normal limiter takes over
self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW))
if ROUTINE_RE.match(line):
self.routine += 1
if 'Loading success' in line: # one summary per start/reload
if LOADED_RE.match(line): # one summary per start/reload
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
% self.routine)
self.routine = 0
+131 -11
View File
@@ -240,13 +240,87 @@ class CapturedScenarioTests(unittest.TestCase):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
evidence = ['ha_panic called', 'Thread 3 is about to kill the process (pid 9).', '[ALERT] foo',
'Current worker (9) exited with code 139 (Segmentation fault)', 'watchdog: thread 2 stuck',
' Thread 2 : id=0x7f', 'FATAL: out of memory', 'segfault at 0', 'got SIGSEGV', 'got SIGABRT']
evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
'[ALERT] (12) : config : unable to load',
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)',
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy',
'[WARNING] (6328) : All workers exited. Exiting... (134)',
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
'* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0']
for l in evidence:
r._emit(l)
self.assertEqual(err.getvalue().count('\n'), len(evidence))
FORGED = [
'GET /FATAL/SIGSEGV/segfault HTTP/1.1',
'[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout',
'[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN',
'[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)',
'Host: Current worker (1) exited with code 134 (Aborted)',
'Host: [ALERT] (1) : forged alert',
'x-forwarded-for: Thread 1 is about to kill the process (pid 1).',
'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1',
'ha_panic watchdog FATAL SIGABRT',
'backend ALERT_Current worker_exited with code 134',
'[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN',
]
def test_forged_keywords_stay_rate_limited(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for l in self.FORGED * 50:
r._emit(l)
self.assertEqual(err.getvalue(), '')
self.assertFalse(r.in_dump)
self.assertEqual(r.suppressed, len(self.FORGED) * 50)
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget
# a forged header in the middle of normal traffic cannot open a block either
r._emit('proxy: Thread 1 is about to kill the process (pid 1).')
r._emit('free text that would be a dump line')
self.assertFalse(r.in_dump)
self.assertEqual(err.getvalue(), '')
def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self):
out, err = io.StringIO(), io.StringIO()
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
r._emit("GET /Loading success HTTP/1.1")
self.assertEqual(r.routine, 0)
self.assertIn('Loading success', out.getvalue())
def test_flood_of_dump_headers_is_capped_by_the_window_budget(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(20000):
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0')
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
n = err.getvalue().count('\n')
self.assertLessEqual(n, relay.CRASH_BUDGET + 5)
self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently
t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again
before = err.getvalue().count('\n')
r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)')
self.assertEqual(err.getvalue().count('\n'), before + 1)
def test_bound_closed_block_cannot_rearm_during_cooldown(self):
out, err, t = io.StringIO(), io.StringIO(), [0.0]
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(relay.DUMP_MAX_LINES + 5):
r._emit('never closed %d' % i)
self.assertFalse(r.in_dump)
r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header
self.assertFalse(r.in_dump)
r._emit('call trace line after the second header')
self.assertNotIn('call trace line after', err.getvalue())
t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again
r._emit('Thread 1 is about to kill the process (pid 3).')
self.assertTrue(r.in_dump)
def test_hostile_and_very_long_lines(self):
r = self.make_relay()
r.poll_once()
@@ -336,17 +410,17 @@ class MutantTests(unittest.TestCase):
self.assertEqual(_scenario_ok(relay), (True, True, True))
def test_mutant_limiter_applied_to_crash_lines(self):
m = self.load_mutant('if CRASH_RE.search(line) or self.in_dump:', 'if False:')
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:')
crash_ok, _, _ = _scenario_ok(m)
self.assertFalse(crash_ok)
def test_mutant_dump_block_not_tracked(self):
m = self.load_mutant('if not self.in_dump and DUMP_START_RE.search(line):', 'if False:')
m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:')
crash_ok, _, _ = _scenario_ok(m)
self.assertFalse(crash_ok)
def test_mutant_collapse_disabled(self):
m = self.load_mutant('if ROUTINE_RE.search(line):', 'if False:')
m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:')
_, collapse_ok, _ = _scenario_ok(m)
self.assertFalse(collapse_ok)
@@ -356,14 +430,60 @@ class MutantTests(unittest.TestCase):
self.assertFalse(prio_ok)
def test_mutant_dump_bound_removed(self):
m = self.load_mutant('DUMP_MAX_LINES = 3000', 'DUMP_MAX_LINES = 10 ** 9')
out, err = io.StringIO(), io.StringIO()
r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ')
err = io.StringIO()
r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(4000):
for i in range(m.DUMP_MAX_LINES if False else 800):
r._emit('never closed %d' % i)
self.assertTrue(r.in_dump) # the mutant never closes: proves the real bound test is meaningful
self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block...
def _forged_run(self, mod):
err = io.StringIO()
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
r.tokens = 0.0
for l in CapturedScenarioTests.FORGED * 20:
r._emit(l)
return err.getvalue()
def test_forged_run_clean_on_real_module(self):
self.assertEqual(self._forged_run(relay), '')
def test_mutant_anchors_removed(self):
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):',
"if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):")
self.assertNotEqual(self._forged_run(m), '')
def _flood(self, mod):
err, t = io.StringIO(), [0.0]
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
r.tokens = 0.0
for i in range(5000):
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
return err.getvalue().count('\n')
def test_flood_capped_on_real_module(self):
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + 5)
def test_mutant_budget_removed(self):
m = self.load_mutant('if self._budget(now):', 'if True:')
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + 5)
def _rearm(self, mod):
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO())
r.tokens = 0.0
r._emit('Thread 1 is about to kill the process (pid 1).')
for i in range(relay.DUMP_MAX_LINES + 5):
r._emit('x %d' % i)
r._emit('Thread 1 is about to kill the process (pid 2).')
return r.in_dump
def test_mutant_cooldown_removed(self):
self.assertFalse(self._rearm(relay))
m = self.load_mutant(' and now >= self.cooldown_until:', ':')
self.assertTrue(self._rearm(m))
def test_mutant_summary_removed(self):
m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines")