From 5e83c8db3b2437714d1a076868cf656b02eec3ef Mon Sep 17 00:00:00 2001 From: jknapp Date: Thu, 13 Aug 2026 14:47:08 -0700 Subject: [PATCH] fix(lsphp): set LSAPI_KEEP_LISTEN=2 to stop idle-exit timing from following the leaked busy counter lsphp's master keeps a `busy` worker counter in a MAP_SHARED page that drifts negative over days of uptime (measured live on whp01: busy=-8 after 6.9 days on arclightcourt.com-01 vs 0..9 on a healthy sibling). php-src's sapi/litespeed/lsapilib.c derives each child's idle-exit grace period from that counter (10 + busy*10, capped by LSAPI_MAX_IDLE) only inside `if (s_keep_listener == 1)`; with busy=-8 that's -70s, so workers exit after ~1s idle instead of 10-30s, no worker ever lingers in accept(), the "don't fork, one's already listening" guard never fires, and the master forks for every connection -- confirmed hitting the max-children ceiling and producing 503s under bot traffic (306 OLS-side ExtConn-timeout/503 errors on the affected site vs 0 on an identical healthy sibling). LSAPI_KEEP_LISTEN=2 skips the `== 1` branch entirely so idle-exit timing falls back to LSAPI_MAX_IDLE (already 30 by default here) instead of the leaked counter. is_enough_free_mem() sits above that branch, not inside it, so the memory-pressure guard is unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/entrypoint-lsphp.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/entrypoint-lsphp.sh b/scripts/entrypoint-lsphp.sh index ff45d47..0e1cec9 100644 --- a/scripts/entrypoint-lsphp.sh +++ b/scripts/entrypoint-lsphp.sh @@ -109,6 +109,32 @@ export PHP_LSAPI_MAX_REQUESTS="${PHP_LSAPI_MAX_REQUESTS:-500}" export LSAPI_MAX_IDLE="${LSAPI_MAX_IDLE:-30}" export LSAPI_EXTRA_CHILDREN="${LSAPI_EXTRA_CHILDREN:-5}" export LSAPI_AVOID_FORK="${LSAPI_AVOID_FORK:-0}" +## LSAPI_KEEP_LISTEN=2 works around a leak in lsphp's own bookkeeping — not a +## setting we're tuning for taste. The master keeps a `busy` worker counter in +## a MAP_SHARED page it shares with its children; measured live on whp01, +## that counter drifts NEGATIVE over days of uptime (arclightcourt.com-01 was +## at busy=-8 after 6.9 days; a healthy sibling sat at 0..9). php-src +## sapi/litespeed/lsapilib.c computes each child's idle-exit grace period as +## `10 + busy*10` seconds (capped by LSAPI_MAX_IDLE) INSIDE +## `if (s_keep_listener == 1)` — with busy=-8 that's `wait_time = -70`, so +## workers exit after ~1s idle instead of 10-30s. No worker then lingers in +## accept(), so the master's "an idle worker is already accepting, don't +## fork" guard never fires and it forks for every single connection — +## observed slamming the hard child ceiling under bot traffic +## (`Reached max children process limit`) and, on rejection, leaving the +## pending connection to rot in the kernel backlog as a 503. Confirmed +## asymmetry: the affected site logged 306 OLS-side `ExtConn timed out` / +## deadlock / `oops! 503` errors where an identically-configured healthy +## sibling logged 0. Restarting the container resets the counter to 0 (it's +## initialised at master start) but it drifts negative again over about a +## week — a reset, not a cure. LSAPI_KEEP_LISTEN=2 skips the `== 1` branch +## entirely, so idle-exit timing is never derived from the leaked counter and +## instead falls straight back to LSAPI_MAX_IDLE above. The is_enough_free_mem() +## memory guard sits immediately above that branch in lsapilib.c and is NOT +## part of it, so it still applies at =2 — this does not trade away the +## memory-pressure protection LSAPI_MAX_IDLE exists for. Still overridable +## (e.g. back to 1) per-container as an escape hatch. +export LSAPI_KEEP_LISTEN="${LSAPI_KEEP_LISTEN:-2}" LSPHP_BIND="${LSPHP_BIND:-0.0.0.0:9000}" ## ---- .user.ini support ---- @@ -130,7 +156,7 @@ LSPHP_BIND="${LSPHP_BIND:-0.0.0.0:9000}" ## .user.ini has not been remediated yet. export LSPHP_ENABLE_USER_INI="${LSPHP_ENABLE_USER_INI:-on}" -echo "Container memory: ${CONTAINER_MEMORY_MB}MB | PHP_LSAPI_CHILDREN=${PHP_LSAPI_CHILDREN} | LSAPI_MAX_IDLE=${LSAPI_MAX_IDLE} | PHPVER=${PHPVER} | bind=${LSPHP_BIND} | user_ini=${LSPHP_ENABLE_USER_INI}" +echo "Container memory: ${CONTAINER_MEMORY_MB}MB | PHP_LSAPI_CHILDREN=${PHP_LSAPI_CHILDREN} | LSAPI_MAX_IDLE=${LSAPI_MAX_IDLE} | LSAPI_KEEP_LISTEN=${LSAPI_KEEP_LISTEN} | PHPVER=${PHPVER} | bind=${LSPHP_BIND} | user_ini=${LSPHP_ENABLE_USER_INI}" ## Validate a numeric value destined for a generated php.ini fragment. ## Sets INI_NUM to the value when it is acceptable, and to "" (plus a WARNING)