diff --git a/CLAUDE.md b/CLAUDE.md index e796618..99a9925 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,4 +179,4 @@ re-reads it on reload, so a failed runtime command must degrade to - gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat - Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/` - HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101) -- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (panic/watchdog dump blocks, `[ALERT]`, `Current worker ... exited with code N`) is NEVER rate-limited and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file +- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (dump blocks opened only by the anchored `Thread N is about to kill` header, `[ALERT] (PID) :`, `Current worker (N) exited with code N`; ANCHORED patterns, so keywords echoed from URLs/Host/server names never match) bypasses the bucket within a budget (1000 lines / 10 min, 60 s cooldown after a bound closes a block) and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file diff --git a/scripts/haproxy-stderr-relay.py b/scripts/haproxy-stderr-relay.py index 1cd26f1..5f70e16 100755 --- a/scripts/haproxy-stderr-relay.py +++ b/scripts/haproxy-stderr-relay.py @@ -17,14 +17,16 @@ Properties * Rare by nature: HAProxy only writes here on start/reload/server state changes/crashes. The per-request access log is NOT here (it goes to the receiver socket), so this does not feed the WHP AI monitor a firehose. -* Crash evidence is NEVER rate-limited and NEVER summarised away: lines that - match CRASH_RE (ha_panic, "Thread N is about to kill", "Current worker ... - exited with code N", [ALERT], watchdog, "Thread N : id=", FATAL, SIGSEGV, - SIGABRT, ...) and every line of a dump block (from a panic/watchdog header to +* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE + ("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is + about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/ + server name never matches) and every line of a dump block (from a panic/watchdog header to the master's "All workers exited" line, bounded by - DUMP_MAX_LINES and DUMP_MAX_SECONDS so a never-closed block cannot disable - the limiter) are forwarded in full even when the bucket is empty. They go to - STDERR. + DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block + can open; blocks open only on the anchored "Thread N is about to kill" header) + are forwarded in full even when the bucket is empty, within a total budget of + CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter + and are counted in the suppressed summary). They go to STDERR. * Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)", config "parsing [...]" warnings, "Former worker (N) exited with code 0", "Initializing new worker", "Loading success", ...) never touches the bucket: @@ -58,30 +60,43 @@ import time PREFIX = 'haproxy-stderr: ' MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output) MAX_PENDING = 65536 # an unterminated line is force-emitted past this size -DUMP_MAX_LINES = 3000 # a dump block never bypasses the limiter longer than this +DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this DUMP_MAX_SECONDS = 30.0 SUMMARY_EVERY = 60.0 -# Crash evidence: forwarded unconditionally (to stderr). Deliberately broad. +# Every pattern is ANCHORED to HAProxy's own message structure (captured from +# real 3.2 crashes): a keyword inside a request path / Host header / server +# name that HAProxy echoes into a warning can never match, because the line +# prefix "[ALERT] (PID) :" is written by HAProxy itself. +# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET. CRASH_RE = re.compile( - r'ha_panic|is about to kill|Current worker|exited with code (?!0\b)|\[ALERT\]' - r'|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=|\bFATAL\b|segfault|SIGSEGV|SIGABRT' - r'|unexpectedly died|bug in haproxy|exit-on-failure|All workers exited|^Hint: when reporting this bug', - re.I) -# A dump block starts at one of these and runs until DUMP_END_RE (or a bound). -DUMP_START_RE = re.compile( - r'ha_panic|is about to kill|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=', re.I) -DUMP_END_RE = re.compile(r'All workers exited') + r'^(?:' + r'\[ALERT\]\s+\(\d+\)\s*:' # any HAProxy alert + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+' + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died' + r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b' + r'|Thread \d+ is about to kill the process' + r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ ' + r')') +# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE. +DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process') +DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b') # A new start/reload cycle also closes a dump block. -CYCLE_RE = re.compile(r'Reloading HAProxy|Initializing new worker|Loading success') +CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)') # Routine per-start/reload noise: counted and collapsed, never rate-limited. ROUTINE_RE = re.compile( + r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:' r'Proxy \S+ stopped \(cumulated conns' r'|config\s*:\s*parsing \[[^\]]*\]\s*:' - r"|Former worker \(\d+\) exited with code 0\b" + r'|Former worker \(\d+\) exited with code 0\b' r'|Reloading HAProxy|Initializing new worker|Loading success' r'|path to executable is|haproxy version is' - r"|Can't load stats-file|'option forwardfor' ignored") + r"|config: Can't load stats-file|'option forwardfor' ignored)") +LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success') +# Total exemption budget: even real-looking evidence cannot flood without bound. +CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70) +CRASH_WINDOW = 600.0 # seconds +DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long def _env_num(name, default, cast=int): @@ -108,6 +123,9 @@ class Relay: self.in_dump = False self.dump_lines = 0 self.dump_since = 0.0 + self.cooldown_until = 0.0 + self.budget_left = CRASH_BUDGET + self.window_start = clock() self.offset = None # None = first pass: start at EOF (do not replay history) self.pending = '' self._last_stream = None @@ -150,31 +168,44 @@ class Relay: % self.suppressed) self.suppressed = 0 - def _end_dump(self, why=None): + def _budget(self, now): + """Take one unit of the crash-evidence budget; False once the window's budget is spent.""" + if now - self.window_start >= CRASH_WINDOW: + self.window_start, self.budget_left = now, CRASH_BUDGET + if self.budget_left > 0: + self.budget_left -= 1 + return True + return False + + def _end_dump(self, now, why=None): self.in_dump = False - if why: + if why: # closed by a bound, not by the master's end line: no re-arming for a while + self.cooldown_until = now + DUMP_COOLDOWN self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why) def _emit(self, line): now = self.clock() if self.in_dump: if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS): - self._end_dump('bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since)) - elif CYCLE_RE.search(line) and not CRASH_RE.search(line): - self._end_dump() - if CRASH_RE.search(line) or self.in_dump: - # crash evidence: bypasses the bucket, the collapse and the summary - if not self.in_dump and DUMP_START_RE.search(line): - self.in_dump, self.dump_lines, self.dump_since = True, 0, now - if self.in_dump: - self.dump_lines += 1 - self._write(self.err, line) - if self.in_dump and DUMP_END_RE.search(line): - self._end_dump() - return - if ROUTINE_RE.search(line): + self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since)) + elif CYCLE_RE.match(line): + self._end_dump(now) + if self.in_dump or CRASH_RE.match(line): + if self._budget(now): + # crash evidence: bypasses the bucket, the collapse and the summary + if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until: + self.in_dump, self.dump_lines, self.dump_since = True, 0, now + if self.in_dump: + self.dump_lines += 1 + self._write(self.err, line) + if self.in_dump and DUMP_END_RE.match(line): + self._end_dump(now) + return + if self.in_dump: # budget spent mid-block: the normal limiter takes over + self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW)) + if ROUTINE_RE.match(line): self.routine += 1 - if 'Loading success' in line: # one summary per start/reload + if LOADED_RE.match(line): # one summary per start/reload self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)' % self.routine) self.routine = 0 diff --git a/scripts/test-haproxy-stderr-relay.py b/scripts/test-haproxy-stderr-relay.py index 31af91f..ba7ddfe 100644 --- a/scripts/test-haproxy-stderr-relay.py +++ b/scripts/test-haproxy-stderr-relay.py @@ -240,13 +240,87 @@ class CapturedScenarioTests(unittest.TestCase): out, err = io.StringIO(), io.StringIO() r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) r.tokens = 0.0 - evidence = ['ha_panic called', 'Thread 3 is about to kill the process (pid 9).', '[ALERT] foo', - 'Current worker (9) exited with code 139 (Segmentation fault)', 'watchdog: thread 2 stuck', - ' Thread 2 : id=0x7f', 'FATAL: out of memory', 'segfault at 0', 'got SIGSEGV', 'got SIGABRT'] + evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)', + '[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM', + '[ALERT] (12) : config : unable to load', + '[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)', + '[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy', + '[WARNING] (6328) : All workers exited. Exiting... (134)', + ' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + '* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0'] for l in evidence: r._emit(l) self.assertEqual(err.getvalue().count('\n'), len(evidence)) + FORGED = [ + 'GET /FATAL/SIGSEGV/segfault HTTP/1.1', + '[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout', + '[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN', + '[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)', + 'Host: Current worker (1) exited with code 134 (Aborted)', + 'Host: [ALERT] (1) : forged alert', + 'x-forwarded-for: Thread 1 is about to kill the process (pid 1).', + 'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1', + 'ha_panic watchdog FATAL SIGABRT', + 'backend ALERT_Current worker_exited with code 134', + '[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN', + ] + + def test_forged_keywords_stay_rate_limited(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in self.FORGED * 50: + r._emit(l) + self.assertEqual(err.getvalue(), '') + self.assertFalse(r.in_dump) + self.assertEqual(r.suppressed, len(self.FORGED) * 50) + self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget + # a forged header in the middle of normal traffic cannot open a block either + r._emit('proxy: Thread 1 is about to kill the process (pid 1).') + r._emit('free text that would be a dump line') + self.assertFalse(r.in_dump) + self.assertEqual(err.getvalue(), '') + + def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit("GET /Loading success HTTP/1.1") + self.assertEqual(r.routine, 0) + self.assertIn('Loading success', out.getvalue()) + + def test_flood_of_dump_headers_is_capped_by_the_window_budget(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(20000): + r._emit('Thread 1 is about to kill the process (pid %d).' % i) + r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0') + r._emit('[WARNING] (1) : All workers exited. Exiting... (134)') + n = err.getvalue().count('\n') + self.assertLessEqual(n, relay.CRASH_BUDGET + 5) + self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently + t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again + before = err.getvalue().count('\n') + r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)') + self.assertEqual(err.getvalue().count('\n'), before + 1) + + def test_bound_closed_block_cannot_rearm_during_cooldown(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 5): + r._emit('never closed %d' % i) + self.assertFalse(r.in_dump) + r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header + self.assertFalse(r.in_dump) + r._emit('call trace line after the second header') + self.assertNotIn('call trace line after', err.getvalue()) + t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again + r._emit('Thread 1 is about to kill the process (pid 3).') + self.assertTrue(r.in_dump) + def test_hostile_and_very_long_lines(self): r = self.make_relay() r.poll_once() @@ -336,17 +410,17 @@ class MutantTests(unittest.TestCase): self.assertEqual(_scenario_ok(relay), (True, True, True)) def test_mutant_limiter_applied_to_crash_lines(self): - m = self.load_mutant('if CRASH_RE.search(line) or self.in_dump:', 'if False:') + m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:') crash_ok, _, _ = _scenario_ok(m) self.assertFalse(crash_ok) def test_mutant_dump_block_not_tracked(self): - m = self.load_mutant('if not self.in_dump and DUMP_START_RE.search(line):', 'if False:') + m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:') crash_ok, _, _ = _scenario_ok(m) self.assertFalse(crash_ok) def test_mutant_collapse_disabled(self): - m = self.load_mutant('if ROUTINE_RE.search(line):', 'if False:') + m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:') _, collapse_ok, _ = _scenario_ok(m) self.assertFalse(collapse_ok) @@ -356,14 +430,60 @@ class MutantTests(unittest.TestCase): self.assertFalse(prio_ok) def test_mutant_dump_bound_removed(self): - m = self.load_mutant('DUMP_MAX_LINES = 3000', 'DUMP_MAX_LINES = 10 ** 9') - out, err = io.StringIO(), io.StringIO() - r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ') + err = io.StringIO() + r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) r.tokens = 0.0 r._emit('Thread 1 is about to kill the process (pid 1).') - for i in range(4000): + for i in range(m.DUMP_MAX_LINES if False else 800): r._emit('never closed %d' % i) - self.assertTrue(r.in_dump) # the mutant never closes: proves the real bound test is meaningful + self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block... + + def _forged_run(self, mod): + err = io.StringIO() + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in CapturedScenarioTests.FORGED * 20: + r._emit(l) + return err.getvalue() + + def test_forged_run_clean_on_real_module(self): + self.assertEqual(self._forged_run(relay), '') + + def test_mutant_anchors_removed(self): + m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', + "if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):") + self.assertNotEqual(self._forged_run(m), '') + + def _flood(self, mod): + err, t = io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + for i in range(5000): + r._emit('Thread 1 is about to kill the process (pid %d).' % i) + r._emit('[WARNING] (1) : All workers exited. Exiting... (134)') + return err.getvalue().count('\n') + + def test_flood_capped_on_real_module(self): + self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + 5) + + def test_mutant_budget_removed(self): + m = self.load_mutant('if self._budget(now):', 'if True:') + self.assertGreater(self._flood(m), relay.CRASH_BUDGET + 5) + + def _rearm(self, mod): + r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO()) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 5): + r._emit('x %d' % i) + r._emit('Thread 1 is about to kill the process (pid 2).') + return r.in_dump + + def test_mutant_cooldown_removed(self): + self.assertFalse(self._rearm(relay)) + m = self.load_mutant(' and now >= self.cooldown_until:', ':') + self.assertTrue(self._rearm(m)) def test_mutant_summary_removed(self): m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines")