2026-11-16 13:02:33 -08:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
|
|
|
|
|
|
|
|
|
|
Why
|
|
|
|
|
---
|
|
|
|
|
HAProxy's master stderr carries what the access log never will: `[ALERT]`
|
|
|
|
|
lines, "Current worker (N) exited with code 134", and the complete
|
|
|
|
|
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
|
|
|
|
|
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
|
|
|
|
|
haproxy-logs volume, so it survives a container recreate). This process tails the
|
|
|
|
|
file and re-emits each line on ITS stderr, prefixed, which is the container's
|
|
|
|
|
stderr: `docker logs`, and through the docker logging driver the host journal
|
|
|
|
|
and Graylog.
|
|
|
|
|
|
|
|
|
|
Properties
|
|
|
|
|
----------
|
|
|
|
|
* Rare by nature: HAProxy only writes here on start/reload/server state
|
|
|
|
|
changes/crashes. The per-request access log is NOT here (it goes to the
|
|
|
|
|
receiver socket), so this does not feed the WHP AI monitor a firehose.
|
2026-11-16 15:08:46 -08:00
|
|
|
* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE
|
|
|
|
|
("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is
|
|
|
|
|
about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/
|
|
|
|
|
server name never matches) and every line of a dump block (from a panic/watchdog header to
|
2026-11-16 15:04:14 -08:00
|
|
|
the master's "All workers exited" line, bounded by
|
2026-11-16 15:08:46 -08:00
|
|
|
DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block
|
|
|
|
|
can open; blocks open only on the anchored "Thread N is about to kill" header)
|
|
|
|
|
are forwarded in full even when the bucket is empty, within a total budget of
|
|
|
|
|
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
|
|
|
|
|
and are counted in the suppressed summary). They go to STDERR.
|
2026-11-16 17:33:27 -08:00
|
|
|
* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!",
|
|
|
|
|
"Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints
|
|
|
|
|
the first at [ALERT]: every tenant container stop/crash/deploy makes them.
|
|
|
|
|
Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own
|
|
|
|
|
"[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI
|
|
|
|
|
monitor through `docker logs haproxy-manager`; rate-limiting them is intended.
|
|
|
|
|
Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend.
|
2026-11-16 15:04:14 -08:00
|
|
|
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
|
|
|
|
|
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
|
|
|
|
|
"Initializing new worker", "Loading success", ...) never touches the bucket:
|
|
|
|
|
it is counted and collapsed into ONE "[relay] reload/start ok: N routine
|
|
|
|
|
lines collapsed" line per reload (plus one on the minute tick for the late
|
|
|
|
|
"Proxy stopped" lines). The full text stays in the stderr file.
|
|
|
|
|
* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST
|
|
|
|
|
lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute
|
|
|
|
|
"[relay] suppressed N lines" summary.
|
|
|
|
|
* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and
|
|
|
|
|
stderr -> PRIORITY 3 (err) and does NOT parse "<N>" prefixes. So benign
|
|
|
|
|
lines (routine, rate-limited remainder, relay summaries) are written to
|
|
|
|
|
STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority.
|
2026-11-16 13:02:33 -08:00
|
|
|
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
|
|
|
|
|
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
|
|
|
|
|
Truncating is safe because HAProxy's fd is O_APPEND.
|
|
|
|
|
* Never blocks or affects HAProxy: it only reads a file.
|
|
|
|
|
|
|
|
|
|
Environment
|
|
|
|
|
-----------
|
|
|
|
|
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
|
|
|
|
|
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
|
|
|
|
|
HAPROXY_STDERR_RELAY_POLL (0.5 s)
|
|
|
|
|
"""
|
|
|
|
|
import os
|
2026-11-16 15:04:14 -08:00
|
|
|
import re
|
2026-11-16 13:02:33 -08:00
|
|
|
import shutil
|
|
|
|
|
import sys
|
|
|
|
|
import time
|
|
|
|
|
|
|
|
|
|
PREFIX = 'haproxy-stderr: '
|
2026-11-16 15:04:14 -08:00
|
|
|
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
|
|
|
|
|
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
|
2026-11-16 15:08:46 -08:00
|
|
|
DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this
|
2026-11-16 15:04:14 -08:00
|
|
|
DUMP_MAX_SECONDS = 30.0
|
|
|
|
|
SUMMARY_EVERY = 60.0
|
|
|
|
|
|
2026-11-16 15:08:46 -08:00
|
|
|
# Every pattern is ANCHORED to HAProxy's own message structure (captured from
|
|
|
|
|
# real 3.2 crashes): a keyword inside a request path / Host header / server
|
|
|
|
|
# name that HAProxy echoes into a warning can never match, because the line
|
|
|
|
|
# prefix "[ALERT] (PID) :" is written by HAProxy itself.
|
|
|
|
|
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
|
2026-11-16 15:04:14 -08:00
|
|
|
CRASH_RE = re.compile(
|
2026-11-16 15:08:46 -08:00
|
|
|
r'^(?:'
|
2026-11-16 17:33:27 -08:00
|
|
|
r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)'
|
2026-11-16 15:08:46 -08:00
|
|
|
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
|
|
|
|
|
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
|
|
|
|
|
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
|
|
|
|
|
r'|Thread \d+ is about to kill the process'
|
|
|
|
|
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
|
|
|
|
|
r')')
|
2026-11-16 17:33:27 -08:00
|
|
|
# Tenant-driven state lines. WHP gives every tenant server `check`, so every
|
|
|
|
|
# tenant container stop/crash/OOM/deploy and every host-wide container restart
|
|
|
|
|
# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash
|
|
|
|
|
# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so
|
|
|
|
|
# they can neither reach priority 3 nor spend the general bucket / crash budgets.
|
|
|
|
|
TENANT_RE = re.compile(
|
|
|
|
|
r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:'
|
|
|
|
|
r"backend '[^']*' has no server available!"
|
|
|
|
|
r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)'
|
|
|
|
|
r'|Health check for (?:backup )?server \S+ '
|
|
|
|
|
r')')
|
2026-11-16 15:08:46 -08:00
|
|
|
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
|
|
|
|
|
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
|
|
|
|
|
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
|
2026-11-16 15:04:14 -08:00
|
|
|
# A new start/reload cycle also closes a dump block.
|
2026-11-16 15:08:46 -08:00
|
|
|
CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)')
|
2026-11-16 15:04:14 -08:00
|
|
|
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
|
|
|
|
|
ROUTINE_RE = re.compile(
|
2026-11-16 15:08:46 -08:00
|
|
|
r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:'
|
2026-11-16 15:04:14 -08:00
|
|
|
r'Proxy \S+ stopped \(cumulated conns'
|
|
|
|
|
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
|
2026-11-16 15:08:46 -08:00
|
|
|
r'|Former worker \(\d+\) exited with code 0\b'
|
2026-11-16 15:04:14 -08:00
|
|
|
r'|Reloading HAProxy|Initializing new worker|Loading success'
|
|
|
|
|
r'|path to executable is|haproxy version is'
|
2026-11-16 15:08:46 -08:00
|
|
|
r"|config: Can't load stats-file|'option forwardfor' ignored)")
|
|
|
|
|
LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success')
|
|
|
|
|
# Total exemption budget: even real-looking evidence cannot flood without bound.
|
|
|
|
|
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
|
|
|
|
|
CRASH_WINDOW = 600.0 # seconds
|
2026-11-16 17:33:27 -08:00
|
|
|
DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that
|
|
|
|
|
# ordinary evidence lines (worker exits, alerts) can never spend
|
|
|
|
|
TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute)
|
|
|
|
|
TENANT_PER_MIN = 30
|
2026-11-16 15:08:46 -08:00
|
|
|
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
|
2026-11-16 13:02:33 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _env_num(name, default, cast=int):
|
|
|
|
|
try:
|
|
|
|
|
v = cast(os.environ.get(name, default))
|
|
|
|
|
return v if v > 0 else default
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return default
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Relay:
|
2026-11-16 15:04:14 -08:00
|
|
|
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None):
|
|
|
|
|
self.path = path
|
|
|
|
|
self.out = out # benign lines (docker journald: PRIORITY 6)
|
|
|
|
|
self.err = err if err is not None else out # crash lines (PRIORITY 3)
|
2026-11-16 13:02:33 -08:00
|
|
|
self.burst, self.per_min = burst, per_min
|
|
|
|
|
self.max_bytes, self.backups = max_bytes, backups
|
|
|
|
|
self.clock = clock
|
|
|
|
|
self.tokens = float(burst)
|
|
|
|
|
self.last = clock()
|
|
|
|
|
self.suppressed = 0
|
2026-11-16 15:04:14 -08:00
|
|
|
self.routine = 0 # collapsed routine lines since the last summary
|
|
|
|
|
self.summary_at = clock()
|
|
|
|
|
self.in_dump = False
|
|
|
|
|
self.dump_lines = 0
|
|
|
|
|
self.dump_since = 0.0
|
2026-11-16 15:08:46 -08:00
|
|
|
self.cooldown_until = 0.0
|
|
|
|
|
self.budget_left = CRASH_BUDGET
|
|
|
|
|
self.window_start = clock()
|
2026-11-16 17:33:27 -08:00
|
|
|
self.dump_budget_left = DUMP_BUDGET
|
|
|
|
|
self.dump_window_start = clock()
|
|
|
|
|
self.tenant_tokens = float(TENANT_BURST)
|
|
|
|
|
self.tenant_last = clock()
|
|
|
|
|
self.tenant_suppressed = 0
|
2026-11-16 13:02:33 -08:00
|
|
|
self.offset = None # None = first pass: start at EOF (do not replay history)
|
|
|
|
|
self.pending = ''
|
2026-11-16 15:04:14 -08:00
|
|
|
self._last_stream = None
|
2026-11-16 13:02:33 -08:00
|
|
|
|
|
|
|
|
def _take(self):
|
|
|
|
|
now = self.clock()
|
|
|
|
|
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
|
|
|
|
|
self.last = now
|
|
|
|
|
if self.tokens >= 1:
|
|
|
|
|
self.tokens -= 1
|
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
2026-11-16 17:33:27 -08:00
|
|
|
def _take_tenant(self):
|
|
|
|
|
now = self.clock()
|
|
|
|
|
self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0)
|
|
|
|
|
self.tenant_last = now
|
|
|
|
|
if self.tenant_tokens >= 1:
|
|
|
|
|
self.tenant_tokens -= 1
|
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
2026-11-16 15:04:14 -08:00
|
|
|
def _write(self, stream, text):
|
|
|
|
|
# keep the two streams in emission order: flush the other one first
|
|
|
|
|
if self._last_stream is not None and self._last_stream is not stream:
|
|
|
|
|
self._last_stream.flush()
|
|
|
|
|
stream.write(PREFIX + text + '\n')
|
|
|
|
|
self._last_stream = stream
|
|
|
|
|
|
|
|
|
|
def _flush_all(self):
|
|
|
|
|
for st in {id(self.out): self.out, id(self.err): self.err}.values():
|
|
|
|
|
try:
|
|
|
|
|
st.flush()
|
|
|
|
|
except Exception:
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def _summaries(self, force=False):
|
|
|
|
|
"""Per-minute summary of what the limiter / collapse withheld."""
|
|
|
|
|
now = self.clock()
|
|
|
|
|
if not force and now - self.summary_at < SUMMARY_EVERY:
|
|
|
|
|
return
|
|
|
|
|
self.summary_at = now
|
|
|
|
|
if self.routine:
|
|
|
|
|
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
|
|
|
|
|
% self.routine)
|
|
|
|
|
self.routine = 0
|
2026-11-16 17:33:27 -08:00
|
|
|
if self.tenant_suppressed:
|
|
|
|
|
self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines '
|
|
|
|
|
'(rate limit; full text in the stderr file)' % self.tenant_suppressed)
|
|
|
|
|
self.tenant_suppressed = 0
|
2026-11-16 15:04:14 -08:00
|
|
|
if self.suppressed:
|
|
|
|
|
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
|
|
|
|
|
% self.suppressed)
|
|
|
|
|
self.suppressed = 0
|
|
|
|
|
|
2026-11-16 17:33:27 -08:00
|
|
|
def _budget(self, now, kind='crash'):
|
|
|
|
|
"""Take one unit of a budget; False once the window's budget is spent.
|
|
|
|
|
|
|
|
|
|
kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget)."""
|
|
|
|
|
if kind == 'dump':
|
|
|
|
|
if now - self.dump_window_start >= CRASH_WINDOW:
|
|
|
|
|
self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET
|
|
|
|
|
if self.dump_budget_left > 0:
|
|
|
|
|
self.dump_budget_left -= 1
|
|
|
|
|
return True
|
|
|
|
|
return False
|
2026-11-16 15:08:46 -08:00
|
|
|
if now - self.window_start >= CRASH_WINDOW:
|
|
|
|
|
self.window_start, self.budget_left = now, CRASH_BUDGET
|
|
|
|
|
if self.budget_left > 0:
|
|
|
|
|
self.budget_left -= 1
|
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
def _end_dump(self, now, why=None):
|
2026-11-16 15:04:14 -08:00
|
|
|
self.in_dump = False
|
2026-11-16 15:08:46 -08:00
|
|
|
if why: # closed by a bound, not by the master's end line: no re-arming for a while
|
|
|
|
|
self.cooldown_until = now + DUMP_COOLDOWN
|
2026-11-16 15:04:14 -08:00
|
|
|
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
|
|
|
|
|
|
2026-11-16 13:02:33 -08:00
|
|
|
def _emit(self, line):
|
2026-11-16 15:04:14 -08:00
|
|
|
now = self.clock()
|
|
|
|
|
if self.in_dump:
|
|
|
|
|
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
|
2026-11-16 15:08:46 -08:00
|
|
|
self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
|
|
|
|
|
elif CYCLE_RE.match(line):
|
|
|
|
|
self._end_dump(now)
|
|
|
|
|
if self.in_dump or CRASH_RE.match(line):
|
2026-11-16 17:33:27 -08:00
|
|
|
is_dump = self.in_dump or bool(DUMP_START_RE.match(line))
|
|
|
|
|
kind = 'dump' if is_dump else 'crash'
|
|
|
|
|
if self._budget(now, kind):
|
2026-11-16 15:08:46 -08:00
|
|
|
# crash evidence: bypasses the bucket, the collapse and the summary
|
|
|
|
|
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
|
|
|
|
|
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
|
|
|
|
|
if self.in_dump:
|
|
|
|
|
self.dump_lines += 1
|
|
|
|
|
self._write(self.err, line)
|
|
|
|
|
if self.in_dump and DUMP_END_RE.match(line):
|
|
|
|
|
self._end_dump(now)
|
|
|
|
|
return
|
|
|
|
|
if self.in_dump: # budget spent mid-block: the normal limiter takes over
|
|
|
|
|
self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW))
|
|
|
|
|
if ROUTINE_RE.match(line):
|
2026-11-16 15:04:14 -08:00
|
|
|
self.routine += 1
|
2026-11-16 15:08:46 -08:00
|
|
|
if LOADED_RE.match(line): # one summary per start/reload
|
2026-11-16 15:04:14 -08:00
|
|
|
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
|
|
|
|
|
% self.routine)
|
|
|
|
|
self.routine = 0
|
|
|
|
|
self._summaries()
|
|
|
|
|
return
|
2026-11-16 17:33:27 -08:00
|
|
|
if TENANT_RE.match(line):
|
|
|
|
|
if self._take_tenant():
|
|
|
|
|
self._write(self.out, line)
|
|
|
|
|
else:
|
|
|
|
|
self.tenant_suppressed += 1
|
|
|
|
|
self._summaries()
|
|
|
|
|
return
|
2026-11-16 13:02:33 -08:00
|
|
|
if self._take():
|
2026-11-16 15:04:14 -08:00
|
|
|
self._write(self.out, line)
|
2026-11-16 13:02:33 -08:00
|
|
|
else:
|
|
|
|
|
self.suppressed += 1
|
2026-11-16 15:04:14 -08:00
|
|
|
self._summaries()
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def _clean(line):
|
|
|
|
|
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
|
|
|
|
|
if len(line) > MAX_LINE:
|
|
|
|
|
line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line)
|
|
|
|
|
return line
|
2026-11-16 13:02:33 -08:00
|
|
|
|
|
|
|
|
def rotate_if_needed(self):
|
|
|
|
|
try:
|
|
|
|
|
if os.path.getsize(self.path) <= self.max_bytes:
|
|
|
|
|
return
|
|
|
|
|
for i in range(self.backups, 1, -1):
|
|
|
|
|
src = '%s.%d' % (self.path, i - 1)
|
|
|
|
|
if os.path.exists(src):
|
|
|
|
|
os.replace(src, '%s.%d' % (self.path, i))
|
|
|
|
|
shutil.copyfile(self.path, self.path + '.1')
|
2026-11-16 17:33:27 -08:00
|
|
|
os.chmod(self.path + '.1', 0o640)
|
2026-11-16 13:02:33 -08:00
|
|
|
with open(self.path, 'r+b') as fh:
|
|
|
|
|
fh.truncate(0)
|
|
|
|
|
self.offset = 0
|
|
|
|
|
except OSError as exc:
|
|
|
|
|
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
|
|
|
|
|
|
|
|
|
|
def poll_once(self):
|
|
|
|
|
try:
|
|
|
|
|
size = os.path.getsize(self.path)
|
|
|
|
|
except OSError:
|
|
|
|
|
if self.offset is None:
|
|
|
|
|
self.offset = 0 # file not created yet: whatever appears is new
|
|
|
|
|
return
|
|
|
|
|
if self.offset is None:
|
|
|
|
|
self.offset = size
|
|
|
|
|
return
|
|
|
|
|
if size < self.offset: # truncated by someone else
|
|
|
|
|
self.offset = 0
|
|
|
|
|
if size == self.offset:
|
|
|
|
|
return
|
|
|
|
|
with open(self.path, 'rb') as fh:
|
|
|
|
|
fh.seek(self.offset)
|
|
|
|
|
chunk = fh.read(1 << 20)
|
|
|
|
|
self.offset += len(chunk)
|
|
|
|
|
text = self.pending + chunk.decode('utf-8', 'replace')
|
|
|
|
|
lines = text.split('\n')
|
|
|
|
|
self.pending = lines.pop() # incomplete trailing line stays buffered
|
2026-11-16 15:04:14 -08:00
|
|
|
if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line
|
|
|
|
|
lines.append(self.pending)
|
|
|
|
|
self.pending = ''
|
2026-11-16 13:02:33 -08:00
|
|
|
for line in lines:
|
2026-11-16 15:04:14 -08:00
|
|
|
line = self._clean(line)
|
2026-11-16 13:02:33 -08:00
|
|
|
if line:
|
|
|
|
|
self._emit(line)
|
2026-11-16 15:04:14 -08:00
|
|
|
self._summaries()
|
|
|
|
|
self._flush_all()
|
|
|
|
|
|
|
|
|
|
def tick(self):
|
|
|
|
|
"""Idle-time housekeeping: emit the per-minute summaries even when no line arrives."""
|
|
|
|
|
self._summaries()
|
|
|
|
|
self._flush_all()
|
2026-11-16 13:02:33 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
2026-11-16 15:04:14 -08:00
|
|
|
relay = Relay(path, sys.stdout,
|
2026-11-16 13:02:33 -08:00
|
|
|
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
|
|
|
|
|
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
|
|
|
|
|
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
|
|
|
|
|
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
|
2026-11-16 15:04:14 -08:00
|
|
|
err=sys.stderr)
|
2026-11-16 13:02:33 -08:00
|
|
|
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
|
2026-11-16 15:04:14 -08:00
|
|
|
sys.stdout.write('haproxy-stderr-relay: following %s\n' % path)
|
|
|
|
|
sys.stdout.flush()
|
2026-11-16 13:02:33 -08:00
|
|
|
ticks = 0
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
|
|
|
|
relay.poll_once()
|
2026-11-16 15:04:14 -08:00
|
|
|
relay.tick()
|
2026-11-16 13:02:33 -08:00
|
|
|
ticks += 1
|
|
|
|
|
if ticks % 120 == 0:
|
|
|
|
|
relay.rotate_if_needed()
|
|
|
|
|
except Exception as exc: # never die: the supervisor loop restarts us anyway
|
|
|
|
|
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
|
|
|
|
|
time.sleep(poll)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|