From 56404f477791764825dae6070f805d2b8a9fe5c2 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 18:27:39 -0800 Subject: [PATCH 1/4] ci: refuse arm64 images that contain non-arm64 binaries (base-arch guard + ELF verification) The in-house python:3.12-slim mirror tag was still a flattened amd64-only manifest, so the linux/arm64 build ran on an amd64 base and published x86-64 binaries labelled arm64 (haproxy-manager-base:latest, :2026.11.6-rc). - Dockerfile: fail the build when dpkg arch != TARGETARCH (no-op when correct) - scripts/verify-image-arch.py (+ test): ELF e_machine of haproxy, bash, python, psutil .so per platform digest - build-push.yaml: verify amd64 (fatal) and arm64 before assembling; an arm64 mismatch is excluded from the tags and fails the job Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- .gitea/workflows/build-push.yaml | 34 ++++++++++- Dockerfile | 11 ++++ scripts/test-verify-image-arch.py | 43 ++++++++++++++ scripts/verify-image-arch.py | 93 +++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100755 scripts/test-verify-image-arch.py create mode 100755 scripts/verify-image-arch.py diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index 407307d..e3d49f3 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -133,6 +133,29 @@ jobs: PKG_CACHE=${{ steps.pkgcache.outputs.url }} outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true + # Per-platform ELF check BEFORE anything is tagged (2026-11-19: an "arm64" + # entry held x86-64 binaries). amd64 mismatch fails the job and publishes + # nothing. An arm64 mismatch (or a failed check) keeps arm64 OUT of the + # tags (amd64-only publish, as for a failed arm64 build) and the job is + # failed afterwards so it is not mistaken for a good multi-arch publish. + - name: Verify amd64 image binaries are amd64 + env: + DIGEST: ${{ steps.amd64.outputs.digest }} + run: | + set -euo pipefail + python3 -I scripts/verify-image-arch.py "repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base@${DIGEST}" amd64 + + - name: Verify arm64 image binaries are arm64 + id: arm64check + if: steps.arm64.outcome == 'success' + continue-on-error: true + timeout-minutes: 10 + env: + DIGEST: ${{ steps.arm64.outputs.digest }} + run: | + set -euo pipefail + python3 -I scripts/verify-image-arch.py "repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base@${DIGEST}" arm64 + # imagetools needs a NEW buildx. Same pinned, sha256-verified binary as # mirror-base-image.yaml (kept identical on purpose). The assemble step # previously ran on whatever buildx the (possibly ephemeral) runner had @@ -163,7 +186,8 @@ jobs: BUILDKIT_PROGRESS: plain AMD64_DIGEST: ${{ steps.amd64.outputs.digest }} ARM64_DIGEST: ${{ steps.arm64.outputs.digest }} - ARM64_OUTCOME: ${{ steps.arm64.outcome }} + # success only if the build AND the ELF check succeeded (a skipped check = not verified = not attached) + ARM64_OUTCOME: ${{ steps.arm64check.outcome == 'success' && steps.arm64.outcome || 'unverified' }} VERSION: ${{ steps.ver.outputs.version }} TAGS: ${{ steps.tags.outputs.tags }} run: | @@ -209,3 +233,11 @@ jobs: --repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \ --repo ghcr.io/shadowdao/haproxy-manager-base \ "${tag_args[@]}" + + # Tags are already published (amd64-only) if arm64 failed verification; fail + # the job so that is visible instead of reading as a green multi-arch publish. + - name: Fail if arm64 was built but its binaries are not arm64 + if: always() && steps.arm64.outcome == 'success' && steps.arm64check.outcome != 'success' + run: | + echo "::error::arm64 image was built but failed the ELF architecture check; tags were published amd64-only (or not at all)" + exit 1 diff --git a/Dockerfile b/Dockerfile index 85732c2..ecfd9f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,17 @@ # Future improvement: a scheduled Gitea Action that does the above automatically. FROM repo.anhonesthost.net/cloud-hosting-platform/python:3.12-slim +# Base-architecture guard. If the mirrored base tag is a single-platform +# (flattened amd64) manifest instead of a multi-arch index, buildx silently uses +# that amd64 base for a linux/arm64 build, every RUN below executes natively as +# x86-64, and the result is x86-64 binaries labelled arm64 (2026-11-19: +# haproxy-manager-base:latest). TARGETARCH is set by buildx (empty on the legacy +# builder, where the check is skipped). On a correct build this is a no-op. +ARG TARGETARCH +RUN if [ -n "${TARGETARCH}" ] && [ "$(dpkg --print-architecture)" != "${TARGETARCH}" ]; then \ + echo "FATAL: base image is $(dpkg --print-architecture) but this build targets ${TARGETARCH}; the in-house python:3.12-slim mirror is not a multi-arch index (re-run mirror-base-image.yaml)" >&2; exit 1; \ + fi + # image.source is what ghcr.io uses to link the package to a GitHub repo # sidebar; pointing at the public GitHub mirror enables that linking. The # canonical source-of-truth git remote is still Gitea, but Gitea's registry diff --git a/scripts/test-verify-image-arch.py b/scripts/test-verify-image-arch.py new file mode 100755 index 0000000..410c9d1 --- /dev/null +++ b/scripts/test-verify-image-arch.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Tests scripts/verify-image-arch.py against a stub `docker` (no daemon needed).""" +import os, stat, subprocess, sys, tempfile, textwrap + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "verify-image-arch.py") +fails = 0 + +def elf(machine): + return b"\x7fELF\x02\x01\x01" + b"\0" * 11 + machine.to_bytes(2, "little") + b"\0" * 40 + +def run(cfg_arch, file_arch, expect_rc, label, psutil_so=True): + global fails + with tempfile.TemporaryDirectory() as td: + stub = os.path.join(td, "docker") + with open(stub, "w") as f: + f.write(textwrap.dedent(f"""\ + #!/usr/bin/env python3 + import sys, os + a = sys.argv[1:] + E = {elf(file_arch)!r} + if a[0] == "image": print("{cfg_arch}") + elif a[0] == "create": print("cid1") + elif a[0] == "cp": + src, dst = a[-2], a[-1] + if src.endswith("/psutil"): + os.makedirs(dst) + if {psutil_so!r}: open(os.path.join(dst, "_psutil_linux.cpython-312.so"), "wb").write(E) + else: open(dst, "wb").write(E) + """)) + os.chmod(stub, os.stat(stub).st_mode | stat.S_IXUSR) + r = subprocess.run([sys.executable, "-I", SCRIPT, "ref@sha256:x", "arm64"], + env={"DOCKER": stub, "PATH": os.environ["PATH"]}, capture_output=True, text=True) + ok = r.returncode == expect_rc + print(("ok " if ok else "FAIL"), label, "rc=", r.returncode) + if not ok: + fails += 1; print(r.stdout, r.stderr) + +run("arm64", 183, 0, "genuine arm64 passes") +run("arm64", 62, 1, "arm64-labelled x86-64 binaries (the 2026-11-19 bug) fail") +run("amd64", 183, 1, "wrong config architecture fails") +run("arm64", 183, 1, "missing compiled extension fails closed", psutil_so=False) +sys.exit(1 if fails else 0) diff --git a/scripts/verify-image-arch.py b/scripts/verify-image-arch.py new file mode 100755 index 0000000..8d941d3 --- /dev/null +++ b/scripts/verify-image-arch.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Verify that a per-platform image really contains binaries for that platform. + +CI only (not copied into images' runtime use). Usage: + + verify-image-arch.py + +Pulls (normally @sha256: of ONE platform, pushed by +digest and not yet tagged) with --platform linux/, never runs it, and +checks: + * the image config's architecture equals ; + * the ELF e_machine of haproxy, bash, the python interpreter and every + compiled extension of the python stack (psutil *.so) equals . + +Why: 2026-11-19, haproxy-manager-base:latest carried an "arm64" entry whose +binaries were all x86-64 (the in-house python:3.12-slim base was a flattened +amd64-only manifest, so the arm64 build silently ran on an amd64 base). +Manifest/config metadata said arm64; only the ELF headers told the truth. + +Exit 0 = every checked file matches; non-zero = mismatch or could not check +(fail closed). Env DOCKER overrides the docker command (tests use a stub). +""" +import os +import shlex +import struct +import subprocess +import sys +import tempfile + +EM = {"amd64": 62, "arm64": 183} # EM_X86_64, EM_AARCH64 +EM_NAME = {62: "x86-64", 183: "aarch64", 40: "arm", 3: "i386"} +FILES = ["/usr/sbin/haproxy", "/bin/bash", "/usr/local/bin/python3.12"] +EXT_DIR = "/usr/local/lib/python3.12/site-packages/psutil" # compiled (_psutil_linux*.so) + + +def elf_machine(path): + """e_machine of an ELF file, or None if it is not a little-endian ELF.""" + with open(path, "rb") as f: + h = f.read(20) + if len(h) < 20 or h[:4] != b"\x7fELF": + return None + endian = "<" if h[5] == 1 else ">" + return struct.unpack(endian + "H", h[18:20])[0] + + +def docker(*args, check=True): + cmd = shlex.split(os.environ.get("DOCKER", "docker")) + list(args) + return subprocess.run(cmd, check=check, capture_output=True, text=True) + + +def main(argv): + if len(argv) != 3 or argv[2] not in EM: + print("usage: verify-image-arch.py ", file=sys.stderr) + return 2 + ref, arch = argv[1], argv[2] + want = EM[arch] + bad = [] + docker("pull", "--quiet", "--platform", f"linux/{arch}", ref) + cfg = docker("image", "inspect", ref, "--format", "{{.Architecture}}").stdout.strip() + print(f"{ref}: image config architecture={cfg} (expected {arch})") + if cfg != arch: + bad.append(f"config architecture {cfg!r}") + cid = docker("create", "--platform", f"linux/{arch}", ref).stdout.strip() + try: + with tempfile.TemporaryDirectory() as td: + for f in FILES: + dst = os.path.join(td, os.path.basename(f)) + docker("cp", "-L", f"{cid}:{f}", dst) + m = elf_machine(dst) + print(f" {f}: {EM_NAME.get(m, m)}") + if m != want: + bad.append(f"{f} is {EM_NAME.get(m, m)}") + extdir = os.path.join(td, "psutil") + docker("cp", f"{cid}:{EXT_DIR}", extdir) + sos = sorted(os.path.join(extdir, n) for n in os.listdir(extdir) if n.endswith(".so")) + if not sos: + bad.append(f"no compiled extension found in {EXT_DIR}") + for so in sos: + m = elf_machine(so) + print(f" psutil/{os.path.basename(so)}: {EM_NAME.get(m, m)}") + if m != want: + bad.append(f"psutil/{os.path.basename(so)} is {EM_NAME.get(m, m)}") + finally: + docker("rm", cid, check=False) + if bad: + print(f"::error::{ref} is labelled linux/{arch} but: " + "; ".join(bad), file=sys.stderr) + return 1 + print(f"OK: {ref} is genuinely {arch}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 8ab6fdcef044aabbe10bd497ab3e6a60a5856dd8 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 19:30:07 -0800 Subject: [PATCH 2/4] feat: HAPROXY_NBTHREAD per-host thread-count knob (global nbthread, only when set) 2026-11-19 sdbees: with nbthread 4 a thread spun on a server lock held by a TLS health-check thread stalled in a page fault under memory pressure -> watchdog abort of the whole edge. nbthread 1 removes the failure class. HAPROXY_NBTHREAD (container env, integer 1-64) renders `nbthread N` in the global section ONLY when set. Unset/invalid = not rendered: every other host's config is byte-identical to before (verified against main's template, with and without Coraza). Invalid values are ignored with a warning, never rendered. Tests: scripts/test-nbthread-knob.py (parse, render, unset byte-identity vs the template minus the block, no cpu-map/thread-groups), run in the image build; real haproxy -c of nbthread 1 (+SPOE+QUIC) and of a bad value in validate-rendered-config.py. VERSION 2026.11.8 (with the arm64 base guard). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- CLAUDE.md | 1 + Dockerfile | 1 + VERSION | 2 +- haproxy_manager.py | 20 ++++++++ scripts/test-nbthread-knob.py | 80 +++++++++++++++++++++++++++++ scripts/validate-rendered-config.py | 15 +++++- templates/hap_header.tpl | 13 +++++ 7 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 scripts/test-nbthread-knob.py diff --git a/CLAUDE.md b/CLAUDE.md index 2da050c..75e462c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,6 +15,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired//` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`. - **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch"). - **Publish verify gate**: `python3 scripts/test-verify-published-tags.py` - offline (needs `jq`; NOT run in the image build); `scripts/verify-published-tags.sh` is the last build-push.yaml step: every tag in every registry must resolve, carry the assembled platforms, and share ONE index digest. The digest is the sha256 of `imagetools inspect --raw` cross-checked against the text `Digest:` line / bare `--format` output (buildx 0.28 ignores `--format` and prints full text; CI run 5381). +- **Per-host thread count knob**: `python3 scripts/test-nbthread-knob.py` - runs in the image build; `HAPROXY_NBTHREAD` (container env, 1-64; invalid/unset = ignored) renders `nbthread N` in `global` ONLY when set, so every other host's config is byte-identical. Real `haproxy -c` of `nbthread 1` is the `nbthread-1` scenario in `scripts/validate-rendered-config.py`. WHP side: `/etc/whp/edge-nbthread` -> container env at recreate. - **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash". - **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines. - **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`. diff --git a/Dockerfile b/Dockerfile index ecfd9f1..212891d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,6 +162,7 @@ RUN python3 /haproxy/scripts/test-wpadmin-gate.py \ && python3 /haproxy/scripts/test-trusted-proxy-gate.py \ && python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \ && python3 /haproxy/scripts/test-default-splash.py \ + && python3 /haproxy/scripts/test-nbthread-knob.py \ && python3 /haproxy/scripts/test-access-log-receiver.py \ && python3 /haproxy/scripts/test-haproxy-stderr-relay.py \ && python3 /haproxy/scripts/test-haproxy-supervisor.py \ diff --git a/VERSION b/VERSION index 04a6e38..a44d4e0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.11.7 +2026.11.8 diff --git a/haproxy_manager.py b/haproxy_manager.py index 7465ce1..6b4a15a 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -3153,6 +3153,21 @@ def get_or_create_cluster_secret(): logger.error("Failed to get/create cluster-secret: %s", e) return '' + +def parse_nbthread(raw): + """HAPROXY_NBTHREAD -> int 1..64, or None (= do not render `nbthread`). + + Fail-safe: anything that is not a plain positive integer in range is + IGNORED with a warning, never rendered, so a typo in a host file cannot + produce an invalid haproxy.cfg (which would be a total edge outage).""" + raw = (raw or '').strip() + if not raw: + return None + if not (raw.isascii() and raw.isdigit()) or not 1 <= int(raw) <= 64: + logger.warning(f"Ignoring HAPROXY_NBTHREAD={raw!r}: expected an integer 1-64") + return None + return int(raw) + def generate_config(): try: conn = sqlite3.connect(DB_FILE) @@ -3246,10 +3261,15 @@ def generate_config(): splash_raw = os.environ.get('HAPROXY_DEFAULT_SPLASH', '').strip().lower() default_splash = splash_raw not in ('0', 'false', 'no', 'off') + # Optional per-host thread count (global `nbthread`). Unset/invalid = + # not rendered = HAProxy's automatic choice, byte-identical config. + nbthread = parse_nbthread(os.environ.get('HAPROXY_NBTHREAD')) + # Add Haproxy Default Headers default_headers = template_env.get_template('hap_header.tpl').render( cluster_secret = get_or_create_cluster_secret(), syslog_target = syslog_target, + nbthread = nbthread, coraza_spoe_backend = coraza_spoe_backend, ) config_parts.append(default_headers) diff --git a/scripts/test-nbthread-knob.py b/scripts/test-nbthread-knob.py new file mode 100644 index 0000000..85a6a31 --- /dev/null +++ b/scripts/test-nbthread-knob.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""HAPROXY_NBTHREAD knob: parsing, rendering, and the unset-path invariant. + +The invariant that matters: a host that does not set HAPROXY_NBTHREAD gets a +`global` section BYTE-IDENTICAL to what the image rendered before the knob +existed (HAProxy keeps choosing its own thread count). The real `haproxy -c` +of the `nbthread 1` shape is in scripts/validate-rendered-config.py. + + python3 scripts/test-nbthread-knob.py +""" +import os +import re +import sys +import unittest + +MODULE_DIR = os.path.abspath( + os.environ.get('HAPROXY_MANAGER_DIR', + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) +sys.path.insert(0, MODULE_DIR) +import haproxy_manager as hm # noqa: E402 + + +def render(**kw): + return hm.template_env.get_template('hap_header.tpl').render( + cluster_secret='s', syslog_target='/log.sock', + coraza_spoe_backend=kw.pop('coraza_spoe_backend', None), **kw) + + +class ParseNbthread(unittest.TestCase): + def test_unset_and_blank(self): + for raw in (None, '', ' '): + self.assertIsNone(hm.parse_nbthread(raw)) + + def test_valid(self): + self.assertEqual(hm.parse_nbthread('1'), 1) + self.assertEqual(hm.parse_nbthread(' 4\n'), 4) + self.assertEqual(hm.parse_nbthread('64'), 64) + + def test_invalid_is_ignored_never_rendered(self): + for raw in ('0', '65', '-1', 'abc', '1.5', '1 2', '1;x', '١', '0x4', '+2'): + self.assertIsNone(hm.parse_nbthread(raw), raw) + + +class Render(unittest.TestCase): + def test_unset_has_no_nbthread_at_all(self): + for kw in ({}, {'nbthread': None}, {'nbthread': 0}, {'coraza_spoe_backend': 'x:1'}): + self.assertNotIn('nbthread', render(**kw).replace('{{', ''), kw) + + def test_unset_is_byte_identical_to_template_without_the_block(self): + src_path = os.path.join(MODULE_DIR, 'templates', 'hap_header.tpl') + with open(src_path) as fh: + src = fh.read() + stripped, n = re.subn(r'\{%- if nbthread %\}.*?\{%- endif %\}\n', '', src, flags=re.S) + self.assertEqual(n, 1, 'expected exactly one nbthread block in hap_header.tpl') + legacy = hm.template_env.from_string(stripped) + for coraza in (None, 'x:1'): + want = legacy.render(cluster_secret='s', syslog_target='/log.sock', + coraza_spoe_backend=coraza) + self.assertEqual(render(coraza_spoe_backend=coraza), want) + self.assertEqual(render(coraza_spoe_backend=coraza, nbthread=None), want) + + def test_set_renders_one_directive_in_global_only(self): + out = render(nbthread=1) + self.assertEqual(len(re.findall(r'^\s*nbthread 1$', out, re.M)), 1) + glob = out.split('\nresolvers ', 1)[0] + self.assertIn('\n nbthread 1\n', glob) + self.assertLess(out.index('nbthread 1'), out.index('tune.ssl.default-dh-param')) + # nothing else changed: removing the directive restores the unset render + self.assertEqual(out.replace('\n nbthread 1', '', 1).count('nbthread 1'), 0) + + def test_no_cpu_map_or_thread_groups_rendered(self): + for kw in ({}, {'nbthread': 1}): + out = '\n'.join(l for l in render(**kw).splitlines() + if not l.lstrip().startswith('#')) + for bad in ('cpu-map', 'thread-groups', 'nbtgroups'): + self.assertNotIn(bad, out) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/validate-rendered-config.py b/scripts/validate-rendered-config.py index 33658a3..545499f 100644 --- a/scripts/validate-rendered-config.py +++ b/scripts/validate-rendered-config.py @@ -146,6 +146,17 @@ SCENARIOS = ( 'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000', 'HAPROXY_DEFAULT_SPLASH': 'off', }), + # Per-host thread count knob: `nbthread 1` must parse next to everything + # else rendered (limited-quic, QUIC bind, SPOE, stats-file). A bad value + # must be IGNORED (not rendered), never reach haproxy. + ('nbthread-1', { + 'HAPROXY_SUSPENSION_ENABLED': 'true', + 'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000', + 'HAPROXY_NBTHREAD': '1', + }), + ('nbthread-bad', { + 'HAPROXY_NBTHREAD': 'abc', + }), ) @@ -308,7 +319,7 @@ def render(scenario_name, env_overrides, workdir): saved_env = {} for key in ('HAPROXY_SUSPENSION_ENABLED', 'HAPROXY_CORAZA_SPOE_BACKEND', - 'HAPROXY_DEFAULT_SPLASH'): + 'HAPROXY_DEFAULT_SPLASH', 'HAPROXY_NBTHREAD'): saved_env[key] = os.environ.pop(key, None) os.environ.update(env_overrides) @@ -342,6 +353,8 @@ def assert_scenario_branches(scenario_name, config_path, env_overrides): 'HAPROXY_CORAZA_SPOE_BACKEND' in env_overrides), 'splash': ('file /haproxy/errors/default-splash.html', env_overrides.get('HAPROXY_DEFAULT_SPLASH') != 'off'), + 'nbthread': ('\n nbthread 1\n', + env_overrides.get('HAPROXY_NBTHREAD') == '1'), } for label, (needle, should_be_present) in expected.items(): present = needle in text diff --git a/templates/hap_header.tpl b/templates/hap_header.tpl index f4fef85..9699d46 100644 --- a/templates/hap_header.tpl +++ b/templates/hap_header.tpl @@ -40,6 +40,19 @@ global user haproxy group haproxy daemon +{%- if nbthread %} + + # PER-HOST THREAD COUNT. Only emitted when HAPROXY_NBTHREAD is set on the + # container (WHP: /etc/whp/edge-nbthread -> container-manager.sh -> env), + # so a host that does not set it renders byte-identically to before and + # HAProxy keeps choosing automatically (all CPUs in its affinity mask). + # Why a host would set 1: 2026-11-19 sdbees, thread 1 spun on a server + # lock held by a TLS health-check thread stalled in a page fault under + # memory pressure -> watchdog abort (whole edge down). With one thread + # there is no other lock holder, so the failure class cannot occur. + # Do NOT combine with cpu-map/thread-groups here: neither is rendered. + nbthread {{ nbthread }} +{%- endif %} # SSL and Performance tune.ssl.default-dh-param 2048 From 4cb4ad39f81a30fb5cb3b74f4f4b0626b4e77ed9 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 19:45:04 -0800 Subject: [PATCH 3/4] docs: nbthread comment names the real plumbing (lib-edge-container.sh) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- CLAUDE.md | 2 +- templates/hap_header.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 75e462c..941a8ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired//` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`. - **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch"). - **Publish verify gate**: `python3 scripts/test-verify-published-tags.py` - offline (needs `jq`; NOT run in the image build); `scripts/verify-published-tags.sh` is the last build-push.yaml step: every tag in every registry must resolve, carry the assembled platforms, and share ONE index digest. The digest is the sha256 of `imagetools inspect --raw` cross-checked against the text `Digest:` line / bare `--format` output (buildx 0.28 ignores `--format` and prints full text; CI run 5381). -- **Per-host thread count knob**: `python3 scripts/test-nbthread-knob.py` - runs in the image build; `HAPROXY_NBTHREAD` (container env, 1-64; invalid/unset = ignored) renders `nbthread N` in `global` ONLY when set, so every other host's config is byte-identical. Real `haproxy -c` of `nbthread 1` is the `nbthread-1` scenario in `scripts/validate-rendered-config.py`. WHP side: `/etc/whp/edge-nbthread` -> container env at recreate. +- **Per-host thread count knob**: `python3 scripts/test-nbthread-knob.py` - runs in the image build; `HAPROXY_NBTHREAD` (container env, 1-64; invalid/unset = ignored) renders `nbthread N` in `global` ONLY when set, so every other host's config is byte-identical. Real `haproxy -c` of `nbthread 1` is the `nbthread-1` scenario in `scripts/validate-rendered-config.py`. WHP side: `/etc/whp/edge-nbthread` -> `lib-edge-container.sh` -> `docker run -e` at recreate. - **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash". - **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines. - **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`. diff --git a/templates/hap_header.tpl b/templates/hap_header.tpl index 9699d46..26958e4 100644 --- a/templates/hap_header.tpl +++ b/templates/hap_header.tpl @@ -43,7 +43,7 @@ global {%- if nbthread %} # PER-HOST THREAD COUNT. Only emitted when HAPROXY_NBTHREAD is set on the - # container (WHP: /etc/whp/edge-nbthread -> container-manager.sh -> env), + # container (WHP: /etc/whp/edge-nbthread -> lib-edge-container.sh -> docker run -e), # so a host that does not set it renders byte-identically to before and # HAProxy keeps choosing automatically (all CPUs in its affinity mask). # Why a host would set 1: 2026-11-19 sdbees, thread 1 spun on a server From f0a737832c9d6319367c27a7c20448d7aee12a7d Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 19 Nov 2026 20:20:48 -0800 Subject: [PATCH 4/4] test: scale the cert-scripts subprocess timeout for emulated (non-x86) builds With python:3.12-slim now a real multi-arch index, the arm64 leg really runs the image gate under QEMU (~30x slower). test-cert-scripts.py took 229 s there and its 200-iteration bash loop exceeded a flat 120 s timeout (CI run 5756), and the whole arm64 step then hit its 30-minute ceiling (run 5758). x86 keeps 120 s; other machines get 900 s. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0185ukXi4YX5FxJGqhi6qab6 --- scripts/test-cert-scripts.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/test-cert-scripts.py b/scripts/test-cert-scripts.py index 2bf412f..d4f4473 100755 --- a/scripts/test-cert-scripts.py +++ b/scripts/test-cert-scripts.py @@ -54,6 +54,7 @@ destructive there; see _cleanup_tmp(). """ import os +import platform import re import shutil import stat @@ -63,6 +64,12 @@ import textwrap import time import unittest +# Subprocess ceiling. The arm64 image gate runs under QEMU (~30x slower: this +# file took 8 s natively and 229 s emulated, and the 200-iteration bash loop in +# test_validation_of_a_live_pem_being_republished_is_not_spurious then exceeded +# a flat 120 s, failing the arm64 leg of every build). Native x86 keeps 120 s. +SUBPROCESS_TIMEOUT = 120 if platform.machine() in ('x86_64', 'AMD64') else 900 + MODULE_DIR = os.path.abspath( os.environ.get('HAPROXY_MANAGER_DIR', os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) @@ -379,7 +386,7 @@ class CertScriptFixture(unittest.TestCase): path = os.path.join(SCRIPTS_DIR, name) self.assertTrue(os.path.exists(path), f'{path} does not exist') return subprocess.run(['bash', path], env=self.env(**env_overrides), - capture_output=True, text=True, timeout=120) + capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT) def bundle_is_valid(self, path): """Ask the shipped library whether HAProxy could use this bundle.""" @@ -964,7 +971,7 @@ class TestCertPublishLibrary(CertScriptFixture): ['bash', '-c', '. "$1"; for i in $(seq 1 200); do cert_bundle_valid "$2" || exit 1; done', '_', LIB, self.live_pem], - env=self.env(), capture_output=True, text=True, timeout=120) + env=self.env(), capture_output=True, text=True, timeout=SUBPROCESS_TIMEOUT) _stop() output = result.stdout + result.stderr