2026-11-16 13:02:33 -08:00
#!/usr/bin/env python3
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
import importlib.util
import io
import os
import tempfile
import unittest
HERE = os . path . dirname ( os . path . abspath ( __file__ ))
spec = importlib . util . spec_from_file_location ( 'relay' , os . path . join ( HERE , 'haproxy-stderr-relay.py' ))
relay = importlib . util . module_from_spec ( spec )
spec . loader . exec_module ( relay )
class Clock :
def __init__ ( self ):
self . t = 0.0
def __call__ ( self ):
return self . t
class RelayTests ( unittest . TestCase ):
def setUp ( self ):
self . dir = tempfile . mkdtemp ( prefix = 'hap-stderr-relay-' )
self . path = os . path . join ( self . dir , 'haproxy-stderr.log' )
open ( self . path , 'w' ) . close ()
self . out = io . StringIO ()
self . clock = Clock ()
def make ( self , burst = 1000 , per_min = 120 , max_bytes = 10 ** 6 , backups = 3 ):
return relay . Relay ( self . path , self . out , burst , per_min , max_bytes , backups , clock = self . clock )
def append ( self , text ):
with open ( self . path , 'a' ) as fh :
fh . write ( text )
def test_history_is_not_replayed_and_new_lines_are_prefixed ( self ):
self . append ( 'old line from a previous container \n ' )
r = self . make ()
r . poll_once ()
self . assertEqual ( self . out . getvalue (), '' )
self . append ( '[ALERT] (10) : Current worker (67) exited with code 134 (Aborted) \n ' )
r . poll_once ()
self . assertEqual ( self . out . getvalue (),
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted) \n ' )
def test_partial_line_is_held_until_complete ( self ):
r = self . make ()
r . poll_once ()
self . append ( 'Thread 1 is abo' )
r . poll_once ()
self . assertEqual ( self . out . getvalue (), '' )
self . append ( 'ut to kill the process. \n ' )
r . poll_once ()
self . assertIn ( 'Thread 1 is about to kill the process.' , self . out . getvalue ())
def test_control_characters_cannot_forge_lines ( self ):
r = self . make ()
r . poll_once ()
self . append ( 'a \x1b [31mb \r\x00 c \n ' )
r . poll_once ()
self . assertEqual ( self . out . getvalue () . count ( ' \n ' ), 1 )
self . assertNotIn ( ' \x1b ' , self . out . getvalue ())
def test_rate_limit_passes_a_full_dump_then_summarises ( self ):
r = self . make ( burst = 200 , per_min = 60 )
r . poll_once ()
self . append ( '' . join ( 'dump line %d \n ' % i for i in range ( 150 )))
r . poll_once ()
self . assertEqual ( self . out . getvalue () . count ( ' \n ' ), 150 )
self . out . truncate ( 0 ), self . out . seek ( 0 )
self . append ( '' . join ( 'flap %d \n ' % i for i in range ( 500 )))
r . poll_once ()
self . assertLessEqual ( self . out . getvalue () . count ( ' \n ' ), 52 ) # ~50 tokens left
self . clock . t += 600 # bucket refills
self . out . truncate ( 0 ), self . out . seek ( 0 )
self . append ( 'after storm \n ' )
r . poll_once ()
2026-11-16 15:04:14 -08:00
r . tick ()
self . assertIn ( '[relay] suppressed ' , self . out . getvalue ())
2026-11-16 13:02:33 -08:00
self . assertIn ( 'after storm' , self . out . getvalue ())
def test_copytruncate_rotation_bounds_the_file_and_keeps_following ( self ):
r = self . make ( max_bytes = 100 , backups = 2 )
r . poll_once ()
self . append ( 'x' * 150 + ' \n ' )
r . poll_once ()
r . rotate_if_needed ()
self . assertEqual ( os . path . getsize ( self . path ), 0 )
self . assertTrue ( os . path . exists ( self . path + '.1' ))
self . append ( 'after rotate \n ' )
r . poll_once ()
self . assertIn ( 'after rotate' , self . out . getvalue ())
def test_missing_file_is_not_an_error ( self ):
os . remove ( self . path )
r = self . make ()
r . poll_once ()
self . append ( 'late \n ' )
r . poll_once ()
self . assertIn ( 'late' , self . out . getvalue ())
2026-11-16 15:04:14 -08:00
# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic ---------------
def reload_block ( pid , n_proxies = 100 ):
"""What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config)."""
L = [ '[NOTICE] ( %d ) : Reloading HAProxy' % pid ,
'[NOTICE] ( %d ) : Initializing new worker ( %d )' % ( pid , pid + 100 ),
'[NOTICE] ( %d ) : haproxy version is 3.2.26-1~bpo13+1' % ( pid + 100 ),
'[NOTICE] ( %d ) : path to executable is /usr/sbin/haproxy' % ( pid + 100 )]
for i in range ( 6 ):
L . append ( "[WARNING] ( %d ) : config : parsing [/etc/haproxy/haproxy.cfg: %d ] : a 'http-request' rule "
"placed after a 'use_backend' rule will still be processed before." % ( pid + 100 , 800 + i ))
L . append ( "[WARNING] ( %d ) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid )
L . append ( '[NOTICE] ( %d ) : Loading success.' % pid )
for i in range ( n_proxies ):
L . append ( '[WARNING] ( %d ) : Proxy be_ %d stopped (cumulated conns: FE: 0, BE: 12).' % ( pid + 50 , i ))
L . append ( '[WARNING] ( %d ) : Former worker ( %d ) exited with code 0 (Exit)' % ( pid , pid + 50 ))
return L
DUMP = [
'Thread 1 is about to kill the process (pid 6631).' ,
'* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ,
' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0' ,
' call trace(15):' ,
' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0' ,
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ,
' lock_hist: R:TASK_WQ U:TASK_WQ' ,
' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ,
' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ,
'' ,
"Hint: when reporting this bug to developers, please check if a core file was" ,
" produced, open it with 'gdb', issue 't a a bt full', check that the" ,
'[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)' ,
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.' ,
'Please check that you are running an up to date and maintained version of haproxy and open a bug report.' ,
'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/' ,
'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html' ,
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM' ,
'[WARNING] (6328) : All workers exited. Exiting... (134)' ,
]
KEYS = ( 'about to kill' , 'Thread 1 :' , 'Thread 2 :' , 'Thread 3 :' , 'Thread 4 :' , 'Current worker' ,
'exited with code 134' , 'exit-on-failure' , 'All workers exited' , 'call trace' , 'ha_dump_backtrace' ,
'bug in haproxy' , 'Please check that you are running' , 'produced, open it with' )
def scenario ( mod , burst = 400 , drained = True ):
"""4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts."""
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = mod . Relay ( '/nonexistent' , out , burst , 120 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
lines = []
for k in range ( 4 ):
lines += reload_block ( 100 * ( k + 1 ))
n_before = len ( lines )
lines += DUMP
for l in lines :
if l . strip (): # poll_once drops blank lines before _emit
r . _emit ( l )
t [ 0 ] += 0.003
r . tick ()
return { 'out' : out . getvalue (), 'err' : err . getvalue (), 'reload_lines' : n_before , 'bucket' : r . tokens , 'r' : r }
class CapturedScenarioTests ( unittest . TestCase ):
def test_drained_bucket_then_panic_forwards_every_crash_line ( self ):
f = scenario ( relay )
self . assertGreater ( f [ 'reload_lines' ], 400 ) # would drain a 400-line bucket
e = f [ 'err' ]
for k in KEYS :
self . assertIn ( k , e , k )
# the whole dump, line for line (blank lines are dropped by design)
got = [ l [ len ( relay . PREFIX ):] for l in e . split ( ' \n ' ) if l ]
self . assertEqual ( got , [ d . rstrip () for d in DUMP if d . strip ()])
def test_dump_is_forwarded_in_full_even_with_an_empty_bucket ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
for l in ( 'benign %d ' % i for i in range ( 10 )):
r . _emit ( l )
self . assertEqual ( out . getvalue (), '' )
for l in DUMP :
if l . strip ():
r . _emit ( l )
self . assertEqual ( err . getvalue () . count ( ' \n ' ), len ([ d for d in DUMP if d . strip ()]))
self . assertEqual ( r . suppressed , 10 )
def test_reload_noise_is_collapsed_not_drained ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
for k in range ( 10 ):
for l in reload_block ( 100 * ( k + 1 )):
r . _emit ( l )
self . assertEqual ( r . tokens , 400.0 ) # bucket untouched by 1000+ noise lines
self . assertEqual ( r . suppressed , 0 )
self . assertEqual ( err . getvalue (), '' )
summ = [ l for l in out . getvalue () . split ( ' \n ' ) if 'reload/start ok' in l ]
self . assertEqual ( len ( summ ), 10 ) # exactly one per reload
self . assertNotIn ( 'Proxy be_' , out . getvalue ())
self . assertNotIn ( 'parsing [' , out . getvalue ())
self . assertLess ( out . getvalue () . count ( ' \n ' ), 15 )
def test_suppressed_lines_are_summarised_per_minute ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 5 , 60 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
for i in range ( 105 ):
r . _emit ( 'flapping server state %d ' % i )
self . assertNotIn ( '[relay] suppressed' , out . getvalue ())
t [ 0 ] += 61
r . tick ()
self . assertRegex ( out . getvalue (), r '\[relay\] suppressed 100 lines' )
self . assertEqual ( r . suppressed , 0 )
self . assertEqual ( err . getvalue (), '' )
def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr ( self ):
# docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "<N>" is NOT parsed
f = scenario ( relay )
self . assertIn ( '[relay] reload/start ok' , f [ 'out' ])
self . assertNotIn ( 'Thread' , f [ 'out' ])
self . assertNotIn ( 'Current worker' , f [ 'out' ])
self . assertNotIn ( '[ALERT]' , f [ 'out' ])
self . assertNotIn ( 'reload/start ok' , f [ 'err' ])
for text in ( f [ 'out' ], f [ 'err' ]):
for l in text . split ( ' \n ' ):
self . assertFalse ( l . startswith ( '<' ) or l . startswith ( relay . PREFIX + '<' ), l )
def test_clean_worker_exit_is_not_crash_evidence ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . _emit ( '[WARNING] (1) : Former worker (5) exited with code 0 (Exit)' )
r . _emit ( '[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)' )
self . assertEqual ( err . getvalue () . count ( ' \n ' ), 1 )
self . assertIn ( 'code 143' , err . getvalue ())
def test_crash_classes_never_limited ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
2026-11-16 15:08:46 -08:00
evidence = [ '[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)' ,
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM' ,
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)' ,
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy' ,
'[WARNING] (6328) : All workers exited. Exiting... (134)' ,
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ,
'* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' ]
2026-11-16 15:04:14 -08:00
for l in evidence :
r . _emit ( l )
self . assertEqual ( err . getvalue () . count ( ' \n ' ), len ( evidence ))
2026-11-16 15:08:46 -08:00
FORGED = [
'GET /FATAL/SIGSEGV/segfault HTTP/1.1' ,
'[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout' ,
'[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN' ,
'[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)' ,
'Host: Current worker (1) exited with code 134 (Aborted)' ,
'Host: [ALERT] (1) : forged alert' ,
'x-forwarded-for: Thread 1 is about to kill the process (pid 1).' ,
'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1' ,
'ha_panic watchdog FATAL SIGABRT' ,
'backend ALERT_Current worker_exited with code 134' ,
'[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN' ,
]
def test_forged_keywords_stay_rate_limited ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
for l in self . FORGED * 50 :
r . _emit ( l )
self . assertEqual ( err . getvalue (), '' )
self . assertFalse ( r . in_dump )
self . assertEqual ( r . suppressed , len ( self . FORGED ) * 50 )
self . assertEqual ( r . budget_left , relay . CRASH_BUDGET ) # none of it touched the crash budget
# a forged header in the middle of normal traffic cannot open a block either
r . _emit ( 'proxy: Thread 1 is about to kill the process (pid 1).' )
r . _emit ( 'free text that would be a dump line' )
self . assertFalse ( r . in_dump )
self . assertEqual ( err . getvalue (), '' )
def test_forged_lines_cannot_be_collapsed_away_or_open_blocks ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . _emit ( "GET /Loading success HTTP/1.1" )
self . assertEqual ( r . routine , 0 )
self . assertIn ( 'Loading success' , out . getvalue ())
def test_flood_of_dump_headers_is_capped_by_the_window_budget ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
r . tokens = 0.0
for i in range ( 20000 ):
r . _emit ( 'Thread 1 is about to kill the process (pid %d ).' % i )
r . _emit ( ' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0' )
r . _emit ( '[WARNING] (1) : All workers exited. Exiting... (134)' )
n = err . getvalue () . count ( ' \n ' )
2026-11-16 17:33:27 -08:00
self . assertLessEqual ( n , relay . CRASH_BUDGET + relay . DUMP_BUDGET + 5 )
2026-11-16 15:08:46 -08:00
self . assertGreater ( r . suppressed , 50000 ) # the rest is counted, not lost silently
t [ 0 ] += relay . CRASH_WINDOW + 1 # a new window: real evidence flows again
before = err . getvalue () . count ( ' \n ' )
r . _emit ( '[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)' )
self . assertEqual ( err . getvalue () . count ( ' \n ' ), before + 1 )
def test_bound_closed_block_cannot_rearm_during_cooldown ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
r . tokens = 0.0
r . _emit ( 'Thread 1 is about to kill the process (pid 1).' )
for i in range ( relay . DUMP_MAX_LINES + 5 ):
r . _emit ( 'never closed %d ' % i )
self . assertFalse ( r . in_dump )
r . _emit ( 'Thread 1 is about to kill the process (pid 2).' ) # immediate new header
self . assertFalse ( r . in_dump )
r . _emit ( 'call trace line after the second header' )
self . assertNotIn ( 'call trace line after' , err . getvalue ())
t [ 0 ] += relay . DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again
r . _emit ( 'Thread 1 is about to kill the process (pid 3).' )
self . assertTrue ( r . in_dump )
2026-11-16 15:04:14 -08:00
def test_hostile_and_very_long_lines ( self ):
r = self . make_relay ()
r . poll_once ()
self . append ( 'A' * 500000 + ' \n ' )
self . append ( ' \x1b ]0;evil \x07 [ALERT] forged \x00\r\n ' )
r . poll_once ()
outs = self . out . getvalue () . split ( ' \n ' )
self . assertTrue ( all ( len ( l ) < relay . MAX_LINE + 200 for l in outs ))
self . assertNotIn ( ' \x1b ' , self . out . getvalue ())
self . assertNotIn ( ' \x00 ' , self . out . getvalue ())
self . assertEqual ( self . out . getvalue () . count ( ' \n ' ), 2 )
def test_unterminated_flood_is_bounded_in_memory ( self ):
r = self . make_relay ()
r . poll_once ()
for _ in range ( 3 ):
self . append ( 'B' * 400000 ) # never a newline
r . poll_once ()
self . assertLessEqual ( len ( r . pending ), relay . MAX_PENDING )
def test_unclosed_dump_block_cannot_disable_the_limiter ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
r . _emit ( 'Thread 1 is about to kill the process (pid 1).' )
for i in range ( relay . DUMP_MAX_LINES + 500 ):
r . _emit ( 'never closed %d ' % i )
self . assertFalse ( r . in_dump )
self . assertLessEqual ( err . getvalue () . count ( ' \n ' ), relay . DUMP_MAX_LINES + 3 )
self . assertGreaterEqual ( r . suppressed , 400 )
def test_dump_block_is_closed_by_time_bound ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
r . tokens = 0.0
r . _emit ( 'Thread 1 is about to kill the process (pid 1).' )
t [ 0 ] += relay . DUMP_MAX_SECONDS + 1
r . _emit ( 'later benign line' )
self . assertFalse ( r . in_dump )
self . assertNotIn ( 'later benign' , err . getvalue ())
2026-11-16 17:33:27 -08:00
# --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence --------
TENANT_LINES = [
"[ALERT] (10) : backend 'be_tenant1' has no server available!" ,
"[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \" Connection refused \" , check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue." ,
"[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue." ,
"[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance." ,
]
def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
for i in range ( 1200 ):
r . _emit ( self . TENANT_LINES [ 0 ])
t [ 0 ] += 0.001
t [ 0 ] += 5.0
for l in DUMP :
if l . strip ():
r . _emit ( l )
got = [ l [ len ( relay . PREFIX ):] for l in err . getvalue () . split ( ' \n ' ) if l ]
self . assertEqual ( got , [ d . rstrip () for d in DUMP if d . strip ()])
self . assertIn ( 'exited with code 134' , err . getvalue ())
self . assertNotIn ( 'exited with code 134' , out . getvalue ())
self . assertNotIn ( 'has no server available' , err . getvalue ())
def test_tenant_lines_are_their_own_rate_limited_stdout_class ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
for i in range ( 1200 ):
r . _emit ( self . TENANT_LINES [ i % len ( self . TENANT_LINES )])
self . assertEqual ( err . getvalue (), '' ) # never PRIORITY 3
self . assertLessEqual ( out . getvalue () . count ( ' \n ' ), relay . TENANT_BURST + 2 )
self . assertGreater ( out . getvalue () . count ( ' \n ' ), 0 ) # but not silently dropped
self . assertEqual ( r . tokens , 400.0 ) # general bucket untouched
self . assertEqual ( r . budget_left , relay . CRASH_BUDGET ) # crash budget untouched
self . assertEqual ( r . dump_budget_left , relay . DUMP_BUDGET )
self . assertGreater ( r . tenant_suppressed , 1000 )
t [ 0 ] += relay . SUMMARY_EVERY + 1
r . tick ()
self . assertRegex ( out . getvalue (), r '\[relay\] suppressed \d+ tenant-driven server/backend state lines' )
self . assertEqual ( r . tenant_suppressed , 0 )
def test_other_alerts_are_not_crash_evidence ( self ):
out , err = io . StringIO (), io . StringIO ()
r = relay . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . _emit ( '[ALERT] (12) : config : unable to load' )
r . _emit ( "[ALERT] (12) : backend 'x' has no server available!" )
self . assertEqual ( err . getvalue (), '' )
self . assertEqual ( out . getvalue () . count ( " \n " ), 2 ) # both reach stdout (rate-limited classes), neither stderr
def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend ( self ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = relay . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
r . tokens = 0.0
for i in range ( relay . CRASH_BUDGET + 500 ): # e.g. a worker crash loop
r . _emit ( '[ALERT] (10) : Current worker ( %d ) exited with code 134 (Aborted)' % i )
self . assertEqual ( r . budget_left , 0 )
before = err . getvalue () . count ( ' \n ' )
for l in DUMP :
if l . strip ():
r . _emit ( l )
self . assertEqual ( err . getvalue () . count ( ' \n ' ) - before , len ([ d for d in DUMP if d . strip ()]))
def test_rotated_backup_is_mode_0640 ( self ):
r = relay . Relay ( self . path , io . StringIO (), 1000 , 120 , 10 , 3 , clock = lambda : 0.0 )
os . chmod ( self . path , 0o644 )
self . append ( 'x' * 100 + ' \n ' )
r . rotate_if_needed ()
self . assertEqual ( os . stat ( self . path + '.1' ) . st_mode & 0o777 , 0o640 )
2026-11-16 15:04:14 -08:00
# shared helpers for the file-driven tests
def setUp ( self ):
self . dir = tempfile . mkdtemp ( prefix = 'hap-stderr-relay-' )
self . path = os . path . join ( self . dir , 'haproxy-stderr.log' )
open ( self . path , 'w' ) . close ()
self . out = io . StringIO ()
def make_relay ( self ):
return relay . Relay ( self . path , self . out , 1000 , 120 , 10 ** 6 , 3 , clock = lambda : 0.0 )
def append ( self , text ):
with open ( self . path , 'a' ) as fh :
fh . write ( text )
# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------
def _scenario_ok ( mod ):
f = scenario ( mod )
got = [ l [ len ( mod . PREFIX ):] for l in f [ 'err' ] . split ( ' \n ' ) if l ]
crash_ok = all ( k in f [ 'err' ] for k in KEYS ) and got == [ d . rstrip () for d in DUMP if d . strip ()]
collapse_ok = f [ 'out' ] . count ( 'reload/start ok' ) == 4 and 'Proxy be_' not in f [ 'out' ] and f [ 'bucket' ] > 300
prio_ok = 'Thread' not in f [ 'out' ] and 'reload/start ok' not in f [ 'err' ]
return crash_ok , collapse_ok , prio_ok
class MutantTests ( unittest . TestCase ):
SRC = os . path . join ( HERE , 'haproxy-stderr-relay.py' )
def load_mutant ( self , old , new ):
with open ( self . SRC ) as fh :
src = fh . read ()
self . assertIn ( old , src , 'mutation target vanished: update the test' )
mutated = src . replace ( old , new , 1 )
self . assertNotEqual ( mutated , src )
d = tempfile . mkdtemp ( prefix = 'hap-relay-mutant-' ) # private copy, the real file is untouched
path = os . path . join ( d , 'mutant.py' )
with open ( path , 'w' ) as fh :
fh . write ( mutated )
sp = importlib . util . spec_from_file_location ( 'mutant' , path )
m = importlib . util . module_from_spec ( sp )
sp . loader . exec_module ( m )
return m
def test_unmutated_copy_passes_everything ( self ):
self . assertEqual ( _scenario_ok ( relay ), ( True , True , True ))
def test_mutant_limiter_applied_to_crash_lines ( self ):
2026-11-16 15:08:46 -08:00
m = self . load_mutant ( 'if self.in_dump or CRASH_RE.match(line):' , 'if False:' )
2026-11-16 15:04:14 -08:00
crash_ok , _ , _ = _scenario_ok ( m )
self . assertFalse ( crash_ok )
def test_mutant_dump_block_not_tracked ( self ):
2026-11-16 15:08:46 -08:00
m = self . load_mutant ( 'if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:' , 'if False:' )
2026-11-16 15:04:14 -08:00
crash_ok , _ , _ = _scenario_ok ( m )
self . assertFalse ( crash_ok )
def test_mutant_collapse_disabled ( self ):
2026-11-16 15:08:46 -08:00
m = self . load_mutant ( 'if ROUTINE_RE.match(line):' , 'if False:' )
2026-11-16 15:04:14 -08:00
_ , collapse_ok , _ = _scenario_ok ( m )
self . assertFalse ( collapse_ok )
def test_mutant_crash_lines_on_stdout ( self ):
m = self . load_mutant ( 'self._write(self.err, line)' , 'self._write(self.out, line)' )
_ , _ , prio_ok = _scenario_ok ( m )
self . assertFalse ( prio_ok )
def test_mutant_dump_bound_removed ( self ):
2026-11-16 15:08:46 -08:00
m = self . load_mutant ( 'DUMP_MAX_LINES = 500 ' , 'DUMP_MAX_LINES = 10 ** 9 ' )
err = io . StringIO ()
r = m . Relay ( '/nonexistent' , io . StringIO (), 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
2026-11-16 15:04:14 -08:00
r . tokens = 0.0
r . _emit ( 'Thread 1 is about to kill the process (pid 1).' )
2026-11-16 15:08:46 -08:00
for i in range ( m . DUMP_MAX_LINES if False else 800 ):
2026-11-16 15:04:14 -08:00
r . _emit ( 'never closed %d ' % i )
2026-11-16 15:08:46 -08:00
self . assertTrue ( r . in_dump ) # real module closes at 500: see test_unclosed_dump_block...
def _forged_run ( self , mod ):
err = io . StringIO ()
r = mod . Relay ( '/nonexistent' , io . StringIO (), 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
for l in CapturedScenarioTests . FORGED * 20 :
r . _emit ( l )
return err . getvalue ()
def test_forged_run_clean_on_real_module ( self ):
self . assertEqual ( self . _forged_run ( relay ), '' )
def test_mutant_anchors_removed ( self ):
m = self . load_mutant ( 'if self.in_dump or CRASH_RE.match(line):' ,
"if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\ d+ : id=', line):" )
self . assertNotEqual ( self . _forged_run ( m ), '' )
def _flood ( self , mod ):
err , t = io . StringIO (), [ 0.0 ]
r = mod . Relay ( '/nonexistent' , io . StringIO (), 1 , 1 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
r . tokens = 0.0
for i in range ( 5000 ):
r . _emit ( 'Thread 1 is about to kill the process (pid %d ).' % i )
r . _emit ( '[WARNING] (1) : All workers exited. Exiting... (134)' )
return err . getvalue () . count ( ' \n ' )
def test_flood_capped_on_real_module ( self ):
2026-11-16 17:33:27 -08:00
self . assertLessEqual ( self . _flood ( relay ), relay . CRASH_BUDGET + relay . DUMP_BUDGET + 5 )
2026-11-16 15:08:46 -08:00
def test_mutant_budget_removed ( self ):
2026-11-16 17:33:27 -08:00
m = self . load_mutant ( 'if self._budget(now, kind):' , 'if True:' )
self . assertGreater ( self . _flood ( m ), relay . CRASH_BUDGET + relay . DUMP_BUDGET + 5 )
2026-11-16 15:08:46 -08:00
def _rearm ( self , mod ):
r = mod . Relay ( '/nonexistent' , io . StringIO (), 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = io . StringIO ())
r . tokens = 0.0
r . _emit ( 'Thread 1 is about to kill the process (pid 1).' )
for i in range ( relay . DUMP_MAX_LINES + 5 ):
r . _emit ( 'x %d ' % i )
r . _emit ( 'Thread 1 is about to kill the process (pid 2).' )
return r . in_dump
def test_mutant_cooldown_removed ( self ):
self . assertFalse ( self . _rearm ( relay ))
m = self . load_mutant ( ' and now >= self.cooldown_until:' , ':' )
self . assertTrue ( self . _rearm ( m ))
2026-11-16 15:04:14 -08:00
def test_mutant_summary_removed ( self ):
m = self . load_mutant ( "self._write(self.out, '[relay] suppressed %d lines" , "self._noop = ('[relay] suppressed %d lines" )
out , t = io . StringIO (), [ 0.0 ]
r = m . Relay ( '/nonexistent' , out , 5 , 60 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = io . StringIO ())
for i in range ( 105 ):
r . _emit ( 'x %d ' % i )
t [ 0 ] += 61
r . tick ()
self . assertNotIn ( '[relay] suppressed' , out . getvalue ())
2026-11-16 17:33:27 -08:00
def _tenant_then_panic_ok ( self , mod ):
out , err , t = io . StringIO (), io . StringIO (), [ 0.0 ]
r = mod . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : t [ 0 ], err = err )
for i in range ( 1200 ):
r . _emit ( "[ALERT] (10) : backend 'be_tenant1' has no server available!" )
t [ 0 ] += 0.001
t [ 0 ] += 5.0
for l in DUMP :
if l . strip ():
r . _emit ( l )
got = [ l [ len ( mod . PREFIX ):] for l in err . getvalue () . split ( ' \n ' ) if l ]
return got == [ d . rstrip () for d in DUMP if d . strip ()] and 'no server available' not in err . getvalue ()
def test_tenant_then_panic_clean_on_real_module ( self ):
self . assertTrue ( self . _tenant_then_panic_ok ( relay ))
def test_mutant_wide_alert_exemption_restored ( self ):
m = self . load_mutant ( " \\ [ALERT \\ ] \\ s+ \\ ( \\ d+ \\ ) \\ s*: \\ s*(?:Current worker" , " \\ [ALERT \\ ] \\ s+ \\ ( \\ d+ \\ ) \\ s*: \\ s*(?:.|Current worker" )
self . assertFalse ( self . _tenant_then_panic_ok ( m ))
def test_mutant_tenant_class_removed ( self ):
m = self . load_mutant ( 'if TENANT_RE.match(line):' , 'if False:' )
self . assertFalse ( self . _tenant_then_panic_ok ( m ) and self . _tenant_stdout_ok ( m ))
def _tenant_stdout_ok ( self , mod ):
out , err = io . StringIO (), io . StringIO ()
r = mod . Relay ( '/nonexistent' , out , 400 , 120 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
for i in range ( 1200 ):
r . _emit ( "[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout" )
return r . tokens == 400.0 and out . getvalue () . count ( ' \n ' ) <= mod . TENANT_BURST + 2
def test_mutant_shared_budget ( self ):
m = self . load_mutant ( "'dump' if is_dump else 'crash'" , "'crash'" )
out , err = io . StringIO (), io . StringIO ()
r = m . Relay ( '/nonexistent' , out , 1 , 1 , 1 << 30 , 3 , clock = lambda : 0.0 , err = err )
r . tokens = 0.0
for i in range ( m . CRASH_BUDGET + 500 ):
r . _emit ( '[ALERT] (10) : Current worker ( %d ) exited with code 134 (Aborted)' % i )
before = err . getvalue () . count ( ' \n ' )
for l in DUMP :
if l . strip ():
r . _emit ( l )
self . assertNotEqual ( err . getvalue () . count ( ' \n ' ) - before , len ([ d for d in DUMP if d . strip ()]))
2026-11-16 13:02:33 -08:00
if __name__ == '__main__' :
unittest . main ()