92 lines
3.8 KiB
Bash
92 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
|||
|
|
# In-container haproxy supervisor (started in the background by start-up.sh).
|
||
|
|
#
|
||
|
|
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
|
||
|
|
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
|
||
|
|
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||
|
|
# policy never fires, and haproxy is down until the external host watchdog
|
||
|
|
# full-restarts the whole container minutes later (dropping every connection).
|
||
|
|
# This loop revives haproxy in place within one interval.
|
||
|
|
#
|
||
|
|
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
|
||
|
|
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
|
||
|
|
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
|
||
|
|
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
|
||
|
|
# most of it detection latency + start; the default is now 2 s (was 15).
|
||
|
|
#
|
||
|
|
# Two backoffs:
|
||
|
|
# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its
|
||
|
|
# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
|
||
|
|
# re-rendering/re-validating the same broken config every 2 s;
|
||
|
|
# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH
|
||
|
|
# seconds (default 10) of that start (a crash loop): extra wait before the
|
||
|
|
# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default
|
||
|
|
# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives
|
||
|
|
# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death
|
||
|
|
# after a long run is still revived within one interval.
|
||
|
|
#
|
||
|
|
# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5,
|
||
|
|
# "abc") falls back to the default instead of killing this subshell under set -e.
|
||
|
|
set -eo pipefail
|
||
|
|
|
||
|
|
_posint() { # _posint VALUE DEFAULT
|
||
|
|
case "$1" in
|
||
|
|
''|*[!0-9]*) echo "$2" ;;
|
||
|
|
*) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)"
|
||
|
|
MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)"
|
||
|
|
QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)"
|
||
|
|
STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)"
|
||
|
|
STEPS=""
|
||
|
|
for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do
|
||
|
|
v="$(_posint "$s" 0)"
|
||
|
|
[ "$v" -ge 1 ] && STEPS="${STEPS} ${v}"
|
||
|
|
done
|
||
|
|
[ -n "${STEPS}" ] || STEPS=" 2 5 10 30"
|
||
|
|
|
||
|
|
# step N (1-based, past the end = last), capped at MAX_BACKOFF
|
||
|
|
_step() {
|
||
|
|
local n="$1" i=0 v=0
|
||
|
|
for s in ${STEPS}; do
|
||
|
|
i=$((i + 1)); v="$s"
|
||
|
|
[ "${i}" -ge "${n}" ] && break
|
||
|
|
done
|
||
|
|
[ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}"
|
||
|
|
echo "${v}"
|
||
|
|
}
|
||
|
|
|
||
|
|
delay="${INTERVAL}"
|
||
|
|
last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet
|
||
|
|
quick=0 # consecutive deaths within QUICK_DEATH of a start
|
||
|
|
while true; do
|
||
|
|
sleep "${delay}"
|
||
|
|
if pidof -q haproxy; then
|
||
|
|
delay="${INTERVAL}"
|
||
|
|
if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then
|
||
|
|
quick=0; last_start=-1
|
||
|
|
fi
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
if [ "${last_start}" -ge 0 ]; then
|
||
|
|
up=$((SECONDS - last_start))
|
||
|
|
if [ "${up}" -lt "${QUICK_DEATH}" ]; then
|
||
|
|
quick=$((quick + 1))
|
||
|
|
wait_s="$(_step "${quick}")"
|
||
|
|
echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2
|
||
|
|
sleep "${wait_s}"
|
||
|
|
pidof -q haproxy && continue
|
||
|
|
elif [ "${up}" -ge "${STABLE}" ]; then
|
||
|
|
quick=0
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
|
||
|
|
delay="${INTERVAL}"
|
||
|
|
last_start="${SECONDS}"
|
||
|
|
else
|
||
|
|
delay=$(( delay * 2 ))
|
||
|
|
[ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}"
|
||
|
|
fi
|
||
|
|
done
|