diff --git a/CLAUDE.md b/CLAUDE.md index 92d9403..e796618 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,4 +179,4 @@ re-reads it on reload, so a failed runtime command must degrade to - gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat - Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/` - HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101) -- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file +- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (panic/watchdog dump blocks, `[ALERT]`, `Current worker ... exited with code N`) is NEVER rate-limited and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay \ No newline at end of file diff --git a/scripts/haproxy-stderr-relay.py b/scripts/haproxy-stderr-relay.py index ab5c9a5..1cd26f1 100755 --- a/scripts/haproxy-stderr-relay.py +++ b/scripts/haproxy-stderr-relay.py @@ -17,9 +17,27 @@ Properties * Rare by nature: HAProxy only writes here on start/reload/server state changes/crashes. The per-request access log is NOT here (it goes to the receiver socket), so this does not feed the WHP AI monitor a firehose. -* Rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST lines burst, - HAPROXY_STDERR_RELAY_PER_MIN refill): a full thread dump (~150 lines) passes, - a flapping-health-check storm is summarised ("N lines suppressed"). +* Crash evidence is NEVER rate-limited and NEVER summarised away: lines that + match CRASH_RE (ha_panic, "Thread N is about to kill", "Current worker ... + exited with code N", [ALERT], watchdog, "Thread N : id=", FATAL, SIGSEGV, + SIGABRT, ...) and every line of a dump block (from a panic/watchdog header to + the master's "All workers exited" line, bounded by + DUMP_MAX_LINES and DUMP_MAX_SECONDS so a never-closed block cannot disable + the limiter) are forwarded in full even when the bucket is empty. They go to + STDERR. +* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)", + config "parsing [...]" warnings, "Former worker (N) exited with code 0", + "Initializing new worker", "Loading success", ...) never touches the bucket: + it is counted and collapsed into ONE "[relay] reload/start ok: N routine + lines collapsed" line per reload (plus one on the minute tick for the late + "Proxy stopped" lines). The full text stays in the stderr file. +* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST + lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute + "[relay] suppressed N lines" summary. +* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and + stderr -> PRIORITY 3 (err) and does NOT parse "" prefixes. So benign + lines (routine, rate-limited remainder, relay summaries) are written to + STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority. * Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES (default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3). Truncating is safe because HAProxy's fd is O_APPEND. @@ -32,11 +50,38 @@ Environment HAPROXY_STDERR_RELAY_POLL (0.5 s) """ import os +import re import shutil import sys import time PREFIX = 'haproxy-stderr: ' +MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output) +MAX_PENDING = 65536 # an unterminated line is force-emitted past this size +DUMP_MAX_LINES = 3000 # a dump block never bypasses the limiter longer than this +DUMP_MAX_SECONDS = 30.0 +SUMMARY_EVERY = 60.0 + +# Crash evidence: forwarded unconditionally (to stderr). Deliberately broad. +CRASH_RE = re.compile( + r'ha_panic|is about to kill|Current worker|exited with code (?!0\b)|\[ALERT\]' + r'|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=|\bFATAL\b|segfault|SIGSEGV|SIGABRT' + r'|unexpectedly died|bug in haproxy|exit-on-failure|All workers exited|^Hint: when reporting this bug', + re.I) +# A dump block starts at one of these and runs until DUMP_END_RE (or a bound). +DUMP_START_RE = re.compile( + r'ha_panic|is about to kill|watchdog|^\s*\*?\s*Thread\s+\d+\s*:\s*id=', re.I) +DUMP_END_RE = re.compile(r'All workers exited') +# A new start/reload cycle also closes a dump block. +CYCLE_RE = re.compile(r'Reloading HAProxy|Initializing new worker|Loading success') +# Routine per-start/reload noise: counted and collapsed, never rate-limited. +ROUTINE_RE = re.compile( + r'Proxy \S+ stopped \(cumulated conns' + r'|config\s*:\s*parsing \[[^\]]*\]\s*:' + r"|Former worker \(\d+\) exited with code 0\b" + r'|Reloading HAProxy|Initializing new worker|Loading success' + r'|path to executable is|haproxy version is' + r"|Can't load stats-file|'option forwardfor' ignored") def _env_num(name, default, cast=int): @@ -48,16 +93,24 @@ def _env_num(name, default, cast=int): class Relay: - def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic): - self.path, self.out = path, out + def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None): + self.path = path + self.out = out # benign lines (docker journald: PRIORITY 6) + self.err = err if err is not None else out # crash lines (PRIORITY 3) self.burst, self.per_min = burst, per_min self.max_bytes, self.backups = max_bytes, backups self.clock = clock self.tokens = float(burst) self.last = clock() self.suppressed = 0 + self.routine = 0 # collapsed routine lines since the last summary + self.summary_at = clock() + self.in_dump = False + self.dump_lines = 0 + self.dump_since = 0.0 self.offset = None # None = first pass: start at EOF (do not replay history) self.pending = '' + self._last_stream = None def _take(self): now = self.clock() @@ -68,14 +121,77 @@ class Relay: return True return False + def _write(self, stream, text): + # keep the two streams in emission order: flush the other one first + if self._last_stream is not None and self._last_stream is not stream: + self._last_stream.flush() + stream.write(PREFIX + text + '\n') + self._last_stream = stream + + def _flush_all(self): + for st in {id(self.out): self.out, id(self.err): self.err}.values(): + try: + st.flush() + except Exception: + pass + + def _summaries(self, force=False): + """Per-minute summary of what the limiter / collapse withheld.""" + now = self.clock() + if not force and now - self.summary_at < SUMMARY_EVERY: + return + self.summary_at = now + if self.routine: + self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)' + % self.routine) + self.routine = 0 + if self.suppressed: + self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)' + % self.suppressed) + self.suppressed = 0 + + def _end_dump(self, why=None): + self.in_dump = False + if why: + self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why) + def _emit(self, line): + now = self.clock() + if self.in_dump: + if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS): + self._end_dump('bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since)) + elif CYCLE_RE.search(line) and not CRASH_RE.search(line): + self._end_dump() + if CRASH_RE.search(line) or self.in_dump: + # crash evidence: bypasses the bucket, the collapse and the summary + if not self.in_dump and DUMP_START_RE.search(line): + self.in_dump, self.dump_lines, self.dump_since = True, 0, now + if self.in_dump: + self.dump_lines += 1 + self._write(self.err, line) + if self.in_dump and DUMP_END_RE.search(line): + self._end_dump() + return + if ROUTINE_RE.search(line): + self.routine += 1 + if 'Loading success' in line: # one summary per start/reload + self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)' + % self.routine) + self.routine = 0 + self._summaries() + return if self._take(): - if self.suppressed: - self.out.write(PREFIX + '[relay] %d line(s) suppressed by rate limit\n' % self.suppressed) - self.suppressed = 0 - self.out.write(PREFIX + line + '\n') + self._write(self.out, line) else: self.suppressed += 1 + self._summaries() + + @staticmethod + def _clean(line): + line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip() + if len(line) > MAX_LINE: + line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line) + return line def rotate_if_needed(self): try: @@ -113,28 +229,38 @@ class Relay: text = self.pending + chunk.decode('utf-8', 'replace') lines = text.split('\n') self.pending = lines.pop() # incomplete trailing line stays buffered + if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line + lines.append(self.pending) + self.pending = '' for line in lines: - line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip() + line = self._clean(line) if line: self._emit(line) - self.out.flush() + self._summaries() + self._flush_all() + + def tick(self): + """Idle-time housekeeping: emit the per-minute summaries even when no line arrives.""" + self._summaries() + self._flush_all() def main(): path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log') - relay = Relay(path, sys.stderr, + relay = Relay(path, sys.stdout, _env_num('HAPROXY_STDERR_RELAY_BURST', 400), _env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120), _env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024), _env_num('HAPROXY_STDERR_LOG_BACKUPS', 3), - ) + err=sys.stderr) poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float) - sys.stderr.write('haproxy-stderr-relay: following %s\n' % path) - sys.stderr.flush() + sys.stdout.write('haproxy-stderr-relay: following %s\n' % path) + sys.stdout.flush() ticks = 0 while True: try: relay.poll_once() + relay.tick() ticks += 1 if ticks % 120 == 0: relay.rotate_if_needed() diff --git a/scripts/test-haproxy-stderr-relay.py b/scripts/test-haproxy-stderr-relay.py index 466e4ac..31af91f 100644 --- a/scripts/test-haproxy-stderr-relay.py +++ b/scripts/test-haproxy-stderr-relay.py @@ -77,7 +77,8 @@ class RelayTests(unittest.TestCase): self.out.truncate(0), self.out.seek(0) self.append('after storm\n') r.poll_once() - self.assertIn('suppressed by rate limit', self.out.getvalue()) + r.tick() + self.assertIn('[relay] suppressed ', self.out.getvalue()) self.assertIn('after storm', self.out.getvalue()) def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self): @@ -101,5 +102,279 @@ class RelayTests(unittest.TestCase): self.assertIn('late', self.out.getvalue()) + +# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic --------------- + +def reload_block(pid, n_proxies=100): + """What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config).""" + L = ['[NOTICE] (%d) : Reloading HAProxy' % pid, + '[NOTICE] (%d) : Initializing new worker (%d)' % (pid, pid + 100), + '[NOTICE] (%d) : haproxy version is 3.2.26-1~bpo13+1' % (pid + 100), + '[NOTICE] (%d) : path to executable is /usr/sbin/haproxy' % (pid + 100)] + for i in range(6): + L.append("[WARNING] (%d) : config : parsing [/etc/haproxy/haproxy.cfg:%d] : a 'http-request' rule " + "placed after a 'use_backend' rule will still be processed before." % (pid + 100, 800 + i)) + L.append("[WARNING] (%d) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid) + L.append('[NOTICE] (%d) : Loading success.' % pid) + for i in range(n_proxies): + L.append('[WARNING] (%d) : Proxy be_%d stopped (cumulated conns: FE: 0, BE: 12).' % (pid + 50, i)) + L.append('[WARNING] (%d) : Former worker (%d) exited with code 0 (Exit)' % (pid, pid + 50)) + return L + + +DUMP = [ + 'Thread 1 is about to kill the process (pid 6631).', + '* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0', + ' call trace(15):', + ' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0', + ' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' lock_hist: R:TASK_WQ U:TASK_WQ', + ' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + ' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0', + '', + "Hint: when reporting this bug to developers, please check if a core file was", + " produced, open it with 'gdb', issue 't a a bt full', check that the", + '[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)', + '[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.', + 'Please check that you are running an up to date and maintained version of haproxy and open a bug report.', + 'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/', + 'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html', + '[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM', + '[WARNING] (6328) : All workers exited. Exiting... (134)', +] +KEYS = ('about to kill', 'Thread 1 :', 'Thread 2 :', 'Thread 3 :', 'Thread 4 :', 'Current worker', + 'exited with code 134', 'exit-on-failure', 'All workers exited', 'call trace', 'ha_dump_backtrace', + 'bug in haproxy', 'Please check that you are running', 'produced, open it with') + + +def scenario(mod, burst=400, drained=True): + """4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts.""" + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = mod.Relay('/nonexistent', out, burst, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + lines = [] + for k in range(4): + lines += reload_block(100 * (k + 1)) + n_before = len(lines) + lines += DUMP + for l in lines: + if l.strip(): # poll_once drops blank lines before _emit + r._emit(l) + t[0] += 0.003 + r.tick() + return {'out': out.getvalue(), 'err': err.getvalue(), 'reload_lines': n_before, 'bucket': r.tokens, 'r': r} + + +class CapturedScenarioTests(unittest.TestCase): + def test_drained_bucket_then_panic_forwards_every_crash_line(self): + f = scenario(relay) + self.assertGreater(f['reload_lines'], 400) # would drain a 400-line bucket + e = f['err'] + for k in KEYS: + self.assertIn(k, e, k) + # the whole dump, line for line (blank lines are dropped by design) + got = [l[len(relay.PREFIX):] for l in e.split('\n') if l] + self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()]) + + def test_dump_is_forwarded_in_full_even_with_an_empty_bucket(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + for l in ('benign %d' % i for i in range(10)): + r._emit(l) + self.assertEqual(out.getvalue(), '') + for l in DUMP: + if l.strip(): + r._emit(l) + self.assertEqual(err.getvalue().count('\n'), len([d for d in DUMP if d.strip()])) + self.assertEqual(r.suppressed, 10) + + def test_reload_noise_is_collapsed_not_drained(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err) + for k in range(10): + for l in reload_block(100 * (k + 1)): + r._emit(l) + self.assertEqual(r.tokens, 400.0) # bucket untouched by 1000+ noise lines + self.assertEqual(r.suppressed, 0) + self.assertEqual(err.getvalue(), '') + summ = [l for l in out.getvalue().split('\n') if 'reload/start ok' in l] + self.assertEqual(len(summ), 10) # exactly one per reload + self.assertNotIn('Proxy be_', out.getvalue()) + self.assertNotIn('parsing [', out.getvalue()) + self.assertLess(out.getvalue().count('\n'), 15) + + def test_suppressed_lines_are_summarised_per_minute(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=err) + for i in range(105): + r._emit('flapping server state %d' % i) + self.assertNotIn('[relay] suppressed', out.getvalue()) + t[0] += 61 + r.tick() + self.assertRegex(out.getvalue(), r'\[relay\] suppressed 100 lines') + self.assertEqual(r.suppressed, 0) + self.assertEqual(err.getvalue(), '') + + def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr(self): + # docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "" is NOT parsed + f = scenario(relay) + self.assertIn('[relay] reload/start ok', f['out']) + self.assertNotIn('Thread', f['out']) + self.assertNotIn('Current worker', f['out']) + self.assertNotIn('[ALERT]', f['out']) + self.assertNotIn('reload/start ok', f['err']) + for text in (f['out'], f['err']): + for l in text.split('\n'): + self.assertFalse(l.startswith('<') or l.startswith(relay.PREFIX + '<'), l) + + def test_clean_worker_exit_is_not_crash_evidence(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err) + r._emit('[WARNING] (1) : Former worker (5) exited with code 0 (Exit)') + r._emit('[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)') + self.assertEqual(err.getvalue().count('\n'), 1) + self.assertIn('code 143', err.getvalue()) + + def test_crash_classes_never_limited(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + evidence = ['ha_panic called', 'Thread 3 is about to kill the process (pid 9).', '[ALERT] foo', + 'Current worker (9) exited with code 139 (Segmentation fault)', 'watchdog: thread 2 stuck', + ' Thread 2 : id=0x7f', 'FATAL: out of memory', 'segfault at 0', 'got SIGSEGV', 'got SIGABRT'] + for l in evidence: + r._emit(l) + self.assertEqual(err.getvalue().count('\n'), len(evidence)) + + def test_hostile_and_very_long_lines(self): + r = self.make_relay() + r.poll_once() + self.append('A' * 500000 + '\n') + self.append('\x1b]0;evil\x07 [ALERT] forged\x00\r\n') + r.poll_once() + outs = self.out.getvalue().split('\n') + self.assertTrue(all(len(l) < relay.MAX_LINE + 200 for l in outs)) + self.assertNotIn('\x1b', self.out.getvalue()) + self.assertNotIn('\x00', self.out.getvalue()) + self.assertEqual(self.out.getvalue().count('\n'), 2) + + def test_unterminated_flood_is_bounded_in_memory(self): + r = self.make_relay() + r.poll_once() + for _ in range(3): + self.append('B' * 400000) # never a newline + r.poll_once() + self.assertLessEqual(len(r.pending), relay.MAX_PENDING) + + def test_unclosed_dump_block_cannot_disable_the_limiter(self): + out, err = io.StringIO(), io.StringIO() + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(relay.DUMP_MAX_LINES + 500): + r._emit('never closed %d' % i) + self.assertFalse(r.in_dump) + self.assertLessEqual(err.getvalue().count('\n'), relay.DUMP_MAX_LINES + 3) + self.assertGreaterEqual(r.suppressed, 400) + + def test_dump_block_is_closed_by_time_bound(self): + out, err, t = io.StringIO(), io.StringIO(), [0.0] + r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + t[0] += relay.DUMP_MAX_SECONDS + 1 + r._emit('later benign line') + self.assertFalse(r.in_dump) + self.assertNotIn('later benign', err.getvalue()) + + # shared helpers for the file-driven tests + def setUp(self): + self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-') + self.path = os.path.join(self.dir, 'haproxy-stderr.log') + open(self.path, 'w').close() + self.out = io.StringIO() + + def make_relay(self): + return relay.Relay(self.path, self.out, 1000, 120, 10 ** 6, 3, clock=lambda: 0.0) + + def append(self, text): + with open(self.path, 'a') as fh: + fh.write(text) + + +# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------ + +def _scenario_ok(mod): + f = scenario(mod) + got = [l[len(mod.PREFIX):] for l in f['err'].split('\n') if l] + crash_ok = all(k in f['err'] for k in KEYS) and got == [d.rstrip() for d in DUMP if d.strip()] + collapse_ok = f['out'].count('reload/start ok') == 4 and 'Proxy be_' not in f['out'] and f['bucket'] > 300 + prio_ok = 'Thread' not in f['out'] and 'reload/start ok' not in f['err'] + return crash_ok, collapse_ok, prio_ok + + +class MutantTests(unittest.TestCase): + SRC = os.path.join(HERE, 'haproxy-stderr-relay.py') + + def load_mutant(self, old, new): + with open(self.SRC) as fh: + src = fh.read() + self.assertIn(old, src, 'mutation target vanished: update the test') + mutated = src.replace(old, new, 1) + self.assertNotEqual(mutated, src) + d = tempfile.mkdtemp(prefix='hap-relay-mutant-') # private copy, the real file is untouched + path = os.path.join(d, 'mutant.py') + with open(path, 'w') as fh: + fh.write(mutated) + sp = importlib.util.spec_from_file_location('mutant', path) + m = importlib.util.module_from_spec(sp) + sp.loader.exec_module(m) + return m + + def test_unmutated_copy_passes_everything(self): + self.assertEqual(_scenario_ok(relay), (True, True, True)) + + def test_mutant_limiter_applied_to_crash_lines(self): + m = self.load_mutant('if CRASH_RE.search(line) or self.in_dump:', 'if False:') + crash_ok, _, _ = _scenario_ok(m) + self.assertFalse(crash_ok) + + def test_mutant_dump_block_not_tracked(self): + m = self.load_mutant('if not self.in_dump and DUMP_START_RE.search(line):', 'if False:') + crash_ok, _, _ = _scenario_ok(m) + self.assertFalse(crash_ok) + + def test_mutant_collapse_disabled(self): + m = self.load_mutant('if ROUTINE_RE.search(line):', 'if False:') + _, collapse_ok, _ = _scenario_ok(m) + self.assertFalse(collapse_ok) + + def test_mutant_crash_lines_on_stdout(self): + m = self.load_mutant('self._write(self.err, line)', 'self._write(self.out, line)') + _, _, prio_ok = _scenario_ok(m) + self.assertFalse(prio_ok) + + def test_mutant_dump_bound_removed(self): + m = self.load_mutant('DUMP_MAX_LINES = 3000', 'DUMP_MAX_LINES = 10 ** 9') + out, err = io.StringIO(), io.StringIO() + r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err) + r.tokens = 0.0 + r._emit('Thread 1 is about to kill the process (pid 1).') + for i in range(4000): + r._emit('never closed %d' % i) + self.assertTrue(r.in_dump) # the mutant never closes: proves the real bound test is meaningful + + def test_mutant_summary_removed(self): + m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines") + out, t = io.StringIO(), [0.0] + r = m.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=io.StringIO()) + for i in range(105): + r._emit('x %d' % i) + t[0] += 61 + r.tick() + self.assertNotIn('[relay] suppressed', out.getvalue()) + + if __name__ == '__main__': unittest.main()