"""gunicorn gthread worker that cannot be wedged by silent or slow clients. Incident 2026-10-09 (sdbees), and very likely the 2026-07-07 whp01 one too ------------------------------------------------------------------------- Both :8000 API workers had 7 of 8 pool threads parked in the kernel in ``tcp_recvmsg -> sk_wait_data``: a blocking socket read with no timeout. /health and /api/blocked-ips hung for 40s+, the container stayed "healthy". haproxy_manager.py makes NO in-process outbound network reads (no requests, urllib, http.client, raw socket or ssl client anywhere; every ACME/HAProxy interaction is a subprocess, already bounded since 2026.07.3). The reads that blocked were gunicorn's own reads of the CLIENT socket: gunicorn/workers/gthread.py TConn.init(): self.sock.setblocking(True) Once the main loop sees a connection become readable it hands it to a pool thread and turns the socket fully blocking -- timeout None -- before parsing the request. Any peer that sends a few bytes and then waits (a TLS ClientHello sent to this plain-HTTP port, a port scanner, a half-written request) pins that pool thread forever. :8000 was published on 0.0.0.0 by the deploy (`-p 8000:8000`, which bypasses firewalld) and internet scanners were measured connecting to it. 4 such connections exhaust a worker's pool. gunicorn's --timeout does not catch this: for gthread it only checks that the worker's MAIN thread keeps heart-beating, and the main thread keeps polling happily while every pool thread is stuck. --max-requests does not help either: a worker whose threads never finish a request never reaches the limit. socket.setdefaulttimeout() does not help: setblocking(True) explicitly resets the timeout to None, overriding any default. Stock gthread has two more exhaustion paths on the same port (review F1/F2): * A connection that NEVER sends a byte never becomes readable, so it is never handed to a thread and never gets any deadline -- but it holds one of the worker's ``worker_connections`` (1000) slots. At 1000 the run loop stops polling altogether and only waits on futures; none exist, so even the attacker's FINs are never seen and the worker is wedged PERMANENTLY, still heart-beating, with no request in flight for a watchdog to notice. * Slowloris: one header byte every _InFlight self._inflight_lock = threading.Lock() self._pending = deque() # accepted, not yet readable (FIFO by deadline) # -- (1) client socket timeout ------------------------------------------ def enqueue_req(self, conn): # Same three steps as ThreadWorker.enqueue_req, with the timeout set # right after conn.init() -- init() is what calls setblocking(True) # (== settimeout(None)), and it runs again on every keep-alive reuse, # so the timeout must be re-applied here each time, not at accept(). conn.init() if CLIENT_TIMEOUT > 0: conn.sock.settimeout(CLIENT_TIMEOUT) fs = self.tpool.submit(self.handle, conn) self._wrap_future(fs, conn) # -- (2) first-byte deadline for never-readable connections --------------- def accept(self, server, listener): # ThreadWorker.accept (23.0.0) plus: a deadline, and the conn is # tracked in self._pending so murder_keepalived() can reap it. try: sock, client = listener.accept() conn = TConn(self.cfg, sock, client, server) conn.timeout = time.time() + FIRST_BYTE_TIMEOUT self.nr_conns += 1 with self._lock: self._pending.append(conn) self.poller.register(conn.sock, selectors.EVENT_READ, partial(self.on_client_socket_readable, conn)) except OSError as e: if e.errno not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise def on_client_socket_readable(self, conn, client): # ThreadWorker version only knows about self._keep; a first request # comes off self._pending instead. with self._lock: self.poller.unregister(client) queue = self._keep if conn.initialized else self._pending try: queue.remove(conn) except ValueError: # already reaped (race with murder_keepalived) return self.enqueue_req(conn) def murder_keepalived(self): # Runs on every pass of ThreadWorker.run() -- including the branch # taken when nr_conns >= worker_connections, where the poller is not # serviced. Reaping here is what breaks that permanent deadlock. super().murder_keepalived() if FIRST_BYTE_TIMEOUT <= 0: return now = time.time() while True: with self._lock: if not self._pending or self._pending[0].timeout > now: break conn = self._pending.popleft() try: self.poller.unregister(conn.sock) except (OSError, KeyError, ValueError): pass self.nr_conns -= 1 conn.close() # -- (3) header deadline + busy tracking ---------------------------------- def handle(self, conn): ident = threading.get_ident() now = time.monotonic() entry = _InFlight(conn, now, now + HEADER_DEADLINE if HEADER_DEADLINE > 0 else None) with self._inflight_lock: self._inflight[ident] = entry try: return super().handle(conn) finally: with self._inflight_lock: self._inflight.pop(ident, None) def handle_request(self, req, conn): # The parser has produced a complete request line + headers. entry = self._inflight.get(threading.get_ident()) if entry is not None: entry.header_deadline = None return super().handle_request(req, conn) def _sweep_header_deadlines(self): now = time.monotonic() with self._inflight_lock: late = [e for e in self._inflight.values() if e.header_deadline is not None and e.header_deadline < now] for e in late: e.header_deadline = None # one shutdown per connection try: # shutdown, not close: wakes the blocked recv() with EOF # without releasing the fd while another thread is in it. e.conn.sock.shutdown(socket.SHUT_RDWR) except OSError: pass if late: self.log.info('Cut %d client connection(s) still sending request headers ' 'after %ss (HAPROXY_MGR_HEADER_DEADLINE)', len(late), HEADER_DEADLINE) def pool_status(self): now = time.monotonic() with self._inflight_lock: starts = [e.start for e in self._inflight.values()] return { 'pid': os.getpid(), 'threads': self.cfg.threads, 'busy': len(starts), 'oldest_busy_seconds': round(now - min(starts), 1) if starts else 0.0, 'client_timeout_seconds': CLIENT_TIMEOUT, } # -- (4) watchdog --------------------------------------------------------- def _wedged_for(self): """Seconds the pool has been completely wedged (every thread busy in its current request), measured from the YOUNGEST in-flight request. 0 when at least one thread is free.""" with self._inflight_lock: starts = [e.start for e in self._inflight.values()] if len(starts) < self.cfg.threads: return 0.0 return time.monotonic() - max(starts) def _dump_stacks(self): frames = sys._current_frames() names = {t.ident: t.name for t in threading.enumerate()} out = [] for ident, frame in frames.items(): out.append('--- thread %s (%s)\n%s' % ( ident, names.get(ident, '?'), ''.join(traceback.format_stack(frame)))) return '\n'.join(out) def _check_wedged(self): wedged = self._wedged_for() if WEDGE_SECONDS > 0 and wedged > WEDGE_SECONDS: try: self.log.critical( 'WEDGED WORKER (pid %s): all %s threads busy in the same request ' 'for %.0fs > HAPROXY_MGR_WEDGE_SECONDS=%s. Exiting so the master ' 'respawns it. Thread stacks:\n%s', os.getpid(), self.cfg.threads, wedged, WEDGE_SECONDS, self._dump_stacks()) for h in logging.getLogger('gunicorn.error').handlers: try: h.flush() except Exception: pass finally: # os._exit, not SIGTERM: a graceful exit joins the (stuck, # non-daemon) pool threads at interpreter shutdown and would # hang until the master's --timeout escalates to SIGKILL. os._exit(WEDGED_EXIT_CODE) def _monitor(self): """One daemon thread: header-deadline sweep every second, wedge check every WATCHDOG_INTERVAL. Never allowed to die silently.""" last_wedge_check = time.monotonic() while self.alive: time.sleep(MONITOR_TICK) try: if HEADER_DEADLINE > 0: self._sweep_header_deadlines() if time.monotonic() - last_wedge_check >= WATCHDOG_INTERVAL: last_wedge_check = time.monotonic() self._check_wedged() except Exception: try: self.log.exception('haproxy_gunicorn_worker monitor error (continuing)') except Exception: pass def init_process(self): global _current_worker _current_worker = self try: self.log.error_log.addFilter(_QuietClientTimeout()) except AttributeError: pass if gunicorn.__version__ != TESTED_GUNICORN_VERSION: self.log.warning( 'haproxy_gunicorn_worker overrides gthread internals as of gunicorn %s; ' 'running %s -- re-run scripts/test-gunicorn-socket-timeouts.py', TESTED_GUNICORN_VERSION, gunicorn.__version__) threading.Thread(target=self._monitor, name='worker-monitor', daemon=True).start() super().init_process() # runs the event loop; does not return