2026-07-01 09:07:11 -07:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Idempotent haproxy liveness check — driven by the in-container supervisor loop.
|
|
|
|
|
|
|
|
|
|
Why this exists
|
|
|
|
|
---------------
|
|
|
|
|
haproxy runs as a *background child of PID 1* (gunicorn) — it is started once at
|
|
|
|
|
container init (scripts/init.py -> do_initial_setup -> start_haproxy) and then
|
|
|
|
|
left running. Nothing supervises it after that. If the haproxy master process
|
|
|
|
|
dies mid-life (SIGABRT -> exit 134, segfault, or an OOM of the haproxy master),
|
|
|
|
|
the container stays "up" because gunicorn is still PID 1, so Docker's
|
|
|
|
|
`--restart` policy never fires. haproxy then stays down until the *external*
|
|
|
|
|
host watchdog (haproxy-watchdog.sh) notices port 80 is dead for ~3 minutes and
|
|
|
|
|
does a full `docker restart` — which drops every in-flight connection.
|
|
|
|
|
|
|
|
|
|
This script closes that gap: called on a short interval by the supervisor loop
|
|
|
|
|
in start-up.sh, it re-launches haproxy *in place* within one interval.
|
|
|
|
|
|
|
|
|
|
Safety
|
|
|
|
|
------
|
|
|
|
|
start_haproxy() is guarded by `is_process_running('haproxy')` (psutil-based, so
|
|
|
|
|
it works in this container which has no `ps`), so calling this while haproxy is
|
|
|
|
|
healthy is a cheap no-op. It only ever acts when haproxy is genuinely gone.
|
|
|
|
|
"""
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
sys.path.insert(0, '/haproxy')
|
|
|
|
|
import haproxy_manager # noqa: E402 (sys.path manipulation must come first)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
if haproxy_manager.is_process_running('haproxy'):
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
haproxy_manager.logger.warning(
|
|
|
|
|
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
|
|
|
|
|
)
|
2026-11-16 13:02:33 -08:00
|
|
|
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
|
|
|
|
|
# before it vanished (the relay forwards the same text; this ties it to the
|
|
|
|
|
# supervisor event even if the relay is behind).
|
|
|
|
|
try:
|
|
|
|
|
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
|
|
|
|
|
fh.seek(0, 2)
|
|
|
|
|
fh.seek(max(0, fh.tell() - 1500))
|
|
|
|
|
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
|
|
|
|
|
for line in tail:
|
|
|
|
|
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
|
|
|
|
|
except OSError:
|
|
|
|
|
pass
|
2026-07-01 09:07:11 -07:00
|
|
|
# start_haproxy() validates the config (and regenerates it if invalid)
|
|
|
|
|
# before launching, and swallows its own errors, so it will not raise here.
|
|
|
|
|
haproxy_manager.start_haproxy()
|
|
|
|
|
|
|
|
|
|
if haproxy_manager.is_process_running('haproxy'):
|
|
|
|
|
haproxy_manager.logger.info("[haproxy-supervisor] haproxy restarted in place")
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
haproxy_manager.logger.error(
|
|
|
|
|
"[haproxy-supervisor] haproxy restart FAILED — still not running after start_haproxy()"
|
|
|
|
|
)
|
|
|
|
|
return 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
sys.exit(main())
|