diff --git a/Dockerfile b/Dockerfile index feaf5f0..f66654f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,23 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \ org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.licenses="MIT" -RUN apt update -y && apt dist-upgrade -y && apt install socat haproxy cron certbot curl jq net-tools -y && apt clean && rm -rf /var/lib/apt/lists/* +# haproxy is PINNED. It was previously unpinned, so the binary could move under +# us at Debian's timing — an upstream release that rejected our config would have +# broken an unrelated commit's build, or worse, shipped an edge that refuses to +# start (see the `haproxy -c` gate below for why that matters: a config HAProxy +# rejects leaves the container Up with 80/443 unbound and /health still 200). +# +# Pinning does NOT make the gate redundant, and the gate does NOT make pinning +# unnecessary — they compose. Pinned means the version moves deliberately; the +# gate then answers immediately whether the new binary still accepts fleet config. +# It also makes the image reproducible, which it previously was not. +# +# To move it: bump the version here, rebuild, and let the gate verify. If Debian +# security-updates the package (e.g. -1+deb13u4) the build FAILS until this pin is +# updated — that failure is the point, not a bug. Check availability with: +# apt-cache policy haproxy +ARG HAPROXY_VERSION=3.0.11-1+deb13u3 +RUN apt update -y && apt dist-upgrade -y && apt install socat "haproxy=${HAPROXY_VERSION}" cron certbot curl jq net-tools -y && apt-mark hold haproxy && apt clean && rm -rf /var/lib/apt/lists/* WORKDIR /haproxy COPY ./templates /haproxy/templates COPY requirements.txt /haproxy/ @@ -40,11 +56,43 @@ COPY trusted_ips.map /etc/haproxy/trusted_ips.map # volume shadows /etc/haproxy with. COPY cloudflare_ips.list /haproxy/defaults/cloudflare_ips.list COPY trusted_proxies.list /haproxy/defaults/trusted_proxies.list +COPY wpadmin_gate_exempt.list /haproxy/defaults/wpadmin_gate_exempt.list # Place errorfiles outside the volumed path; the HAProxy config references # them by absolute path. COPY errors /haproxy/errors RUN chmod +x /haproxy/scripts/* RUN pip install -r requirements.txt +# --------------------------------------------------------------------------- +# Build gate: no image ships unless the real haproxy binary accepts the config +# this image's templates actually produce. +# +# On 2026-08-14 a template change rendered fine, passed all 13 unit tests, and +# was rejected by HAProxy ("invalid arg 2 in converter 'regsub'"). It was only +# caught because someone built an image by hand and ran `haproxy -c`. Nothing +# in the build or in CI would have stopped it: .gitea/workflows/build-push.yaml +# is checkout -> build -> push, and test-config-rollback.py's "haproxy" is a +# shell stub that only rejects a sentinel token. In production an invalid +# haproxy.cfg means init.py refuses to start HAProxy while the container stays +# Up - ports 80/443 unbound, every site on the host down, /health still 200. +# +# This lives in the Dockerfile rather than in the workflow deliberately: +# * it cannot be skipped, and it protects local `docker build` too; +# * no workflow restructuring (build-push-action builds and pushes in one +# step, so gating in CI would mean splitting build from push); +# * it validates against the EXACT haproxy binary in this image. Line 26 +# installs haproxy unpinned, so that binary moves between builds - this +# turns "the new haproxy rejects our config" from a silent production +# risk into a build failure. +# +# The unit suites run here too. They had never run anywhere automated either, +# and they cost a few seconds. +RUN python3 /haproxy/scripts/test-wpadmin-gate.py \ + && python3 /haproxy/scripts/test-trusted-proxy-gate.py \ + && python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \ + && python3 /haproxy/scripts/test-config-rollback.py \ + && python3 /haproxy/scripts/test-cert-write-safety.py \ + && python3 /haproxy/scripts/test-cert-scripts.py \ + && python3 /haproxy/scripts/validate-rendered-config.py # Create log directories RUN mkdir -p /var/log && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log RUN chmod 755 /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log diff --git a/scripts/start-up.sh b/scripts/start-up.sh index 975bd73..02494d2 100755 --- a/scripts/start-up.sh +++ b/scripts/start-up.sh @@ -27,11 +27,12 @@ mkdir -p /etc/haproxy # existing hosts instead of being permanently shadowed by the volume. # Overwrite it from the baked copy on every start. # -# trusted_proxies.list is OPERATOR DATA: operators add entries directly on -# the server and those must survive restarts/recreates. Seed it from the -# baked copy only when it's missing; never overwrite an existing one. +# trusted_proxies.list and wpadmin_gate_exempt.list are OPERATOR DATA: +# operators add entries directly on the server and those must survive +# restarts/recreates. Seed each from the baked copy only when it's missing; +# never overwrite an existing one. # -# Both branches fall back to an empty file if the baked default is somehow +# All branches fall back to an empty file if the baked default is somehow # absent, because "acl ... -f " is a fatal HAProxy config # error -- the list files must exist unconditionally by the time HAProxy starts. if [ -f /haproxy/defaults/cloudflare_ips.list ]; then @@ -46,6 +47,13 @@ if [ ! -f /etc/haproxy/trusted_proxies.list ]; then : > /etc/haproxy/trusted_proxies.list fi fi +if [ ! -f /etc/haproxy/wpadmin_gate_exempt.list ]; then + if [ -f /haproxy/defaults/wpadmin_gate_exempt.list ]; then + cp /haproxy/defaults/wpadmin_gate_exempt.list /etc/haproxy/wpadmin_gate_exempt.list + else + : > /etc/haproxy/wpadmin_gate_exempt.list + fi +fi cron & diff --git a/scripts/test-wpadmin-gate.py b/scripts/test-wpadmin-gate.py new file mode 100644 index 0000000..16abc91 --- /dev/null +++ b/scripts/test-wpadmin-gate.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python3 +"""Regression tests for the WordPress admin edge gate in hap_listener.tpl. + +Why this file exists +-------------------- +Unauthenticated GETs to /wp-admin/* were reaching PHP, booting WordPress just to +produce a login redirect and exhausting lsphp pools under a distributed +low-and-slow attack. The gate redirects them at the edge instead. + +Two properties are easy to get wrong and invisible to `haproxy -c`: + + * ORDERING. `is_whitelisted` reads var(txn.real_ip). If the rule renders before + the set-var chain, the whitelist evaluates against an unset variable. + * THE ALLOWLIST. wp-login.php loads its OWN css/js from /wp-admin/. Dropping + those entries leaves every login page on the fleet unstyled, while still + returning 200 -- a silent regression. + +A THIRD property, added after an adversarial mutation audit: every assertion +here must be scoped to the CODE, not the surrounding prose. This file's own +comment blocks quote ACL names, rule fragments and even whole rules to explain +them -- which means a bare `assertIn` / `re.search` / `str.index` run over the +raw rendered config passes just as happily when the real rule has been deleted +(or merely commented out) and only its explanation survives. The audit proved +this concretely: commenting out the entire redirect rule, or the +`acl wp_admin_allowed` line, or all five normalizers, left the previous version +of this file at 26/26 PASS. See `rule_lines()` below, and use it (or one of the +guarded helpers built on it) for every assertion about whether a rule exists, +what it says, or where it sits relative to another rule. Do not add a new +`self.cfg.index(...)`, `self.assertIn(x, self.cfg)`, or `re.search(pattern, +self.cfg)` to this file -- none of them can tell code from comment. + +Running +------- + python3 scripts/test-wpadmin-gate.py +""" + +import os +import re +import sys +import logging +import tempfile +import unittest + +MODULE_DIR = os.path.abspath( + os.environ.get('HAPROXY_MANAGER_DIR', + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +) +os.chdir(MODULE_DIR) +sys.path.insert(0, MODULE_DIR) + +_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +_real_file_handler = logging.FileHandler +logging.FileHandler = ( + lambda filename, *a, **kw: _real_file_handler( + os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw) +) + +import haproxy_manager # noqa: E402 + +ALLOWLIST = ('/admin-ajax.php', '/admin-post.php', + '/load-styles.php', '/load-scripts.php') +EXCLUSIONS = ('!wp_admin_allowed', '!wp_admin_asset', '!has_wp_logged_in', + '!wp_gate_exempt', '!is_local', '!is_trusted_ip', '!is_whitelisted') + +# The normalizer set, in the order it MUST render. Decoding has to precede the +# path walkers or "%2e%2e" is decoded to ".." only after path-strip-dotdot has +# already run, leaving the ".." unresolved -- measured against real HAProxy +# 3.0.11, both orders side by side. +NORMALIZERS = ('percent-to-uppercase', + 'percent-decode-unreserved', + 'path-merge-slashes', + 'path-strip-dot', + 'path-strip-dotdot full') + + +def render_listener(): + return haproxy_manager.template_env.get_template('hap_listener.tpl').render( + crt_path='/etc/haproxy/certs', + suspension_enabled=False, + coraza_spoe_backend=None, + ) + + +def render_header(): + return haproxy_manager.template_env.get_template('hap_header.tpl').render( + cluster_secret=None, + ) + + +# --------------------------------------------------------------------------- +# Comment-safe config inspection +# +# Every helper below operates on `rule_positions()`'s output, never on the raw +# rendered string. That is the one rule this whole module exists to enforce on +# itself: a mutation audit found that commenting out a real rule (prefixing it +# with '#', or -- more slyly -- deleting it and appending its own text as a +# TRAILING comment on the line above) left the previous version of these tests +# fully green, because plain `str.index` / `assertIn` / `re.search` over +# `self.cfg` cannot distinguish code from a comment that merely quotes it. +# --------------------------------------------------------------------------- + +def rule_positions(cfg, needle): + """[(comment-stripped line, char offset in cfg)] for every non-comment + line containing `needle`, in document order. + + Two things a bare substring/regex search over `self.cfg` gets wrong, both + fixed here: + + 1. A line that is ENTIRELY a comment (starts with '#' once stripped) is + dropped. This is necessary but not sufficient -- see (2). + 2. A line that MIXES real config with a trailing comment + (`live-code # note`, or the decoy `live-code # was: `) is truncated at the first ' #' before matching, + so text stuffed into a trailing comment cannot masquerade as the + rule itself. A real HAProxy comment always starts at a '#' preceded + by whitespace here -- none of these templates use bare '#' as a + value character -- so this truncation does not clip real rules. + + Returning (line, position) pairs together -- rather than making callers + re-derive one from the other with a second `cfg.index(line)` -- also + avoids a subtler bug: if the same comment-stripped line occurs twice + (e.g. a duplicated rule), re-deriving the position with `str.index` always + finds the FIRST copy regardless of which one you meant. Walking the file + once and recording positions as we go keeps first/last unambiguous. + """ + out = [] + pos = 0 + for raw in cfg.split('\n'): + stripped = raw.strip() + if stripped and not stripped.startswith('#'): + code = stripped.split(' #', 1)[0].rstrip() + if code and needle in code: + out.append((code, pos)) + pos += len(raw) + 1 # +1 for the '\n' split() consumed + return out + + +def rule_lines(cfg, needle): + """Comment-stripped rule lines containing `needle` (text only, no + position). See `rule_positions()` for what this guards against. Every + assertion about a RULE's presence or content must go through this (or + `rule_positions`/the guarded helpers below) -- never a bare + `needle in cfg` or `re.search(pattern, cfg)`. + """ + return [line for line, _ in rule_positions(cfg, needle)] + + +def require_rule(cfg, needle, what=None): + """The single rule line containing `needle`. + + Raises a plain AssertionError naming what was being looked for -- not + IndexError from an unguarded `rule_lines(...)[0]`, and not + `ValueError: substring not found` from a bare `cfg.index(...)` -- when + the rule is missing. A missing rule and a broken test harness must not + look identical in a failure report. + + Raises the same way if `needle` is ambiguous (matches more than one rule + line): silently taking the first match in that case would hide the + ambiguity instead of surfacing it. + """ + label = what or needle + lines = rule_lines(cfg, needle) + if not lines: + raise AssertionError('no rule found for %r (expected: %s)' % (needle, label)) + if len(lines) > 1: + raise AssertionError( + '%r matched %d rule lines, expected exactly one (%s): %r' + % (needle, len(lines), label, lines)) + return lines[0] + + +def require_position(cfg, needle, what=None, last=False): + """(line, char offset) for an ordering assertion, guarded the same way as + `require_rule` -- but tolerant of the needle matching multiple lines + (e.g. a multi-line set-var "chain"), since ordering checks often want the + first or last of several. Pass last=True for the last occurrence. + """ + label = what or needle + positions = rule_positions(cfg, needle) + if not positions: + raise AssertionError( + 'no rule found for %r, cannot check ordering (expected: %s)' % (needle, label)) + return positions[-1] if last else positions[0] + + +class WpAdminGate(unittest.TestCase): + + def setUp(self): + self.cfg = render_listener() + + def test_wp_admin_path_acl_declared(self): + line = require_rule(self.cfg, 'acl wp_admin_path', 'wp_admin_path ACL') + self.assertIn('path_reg', line) + + def test_wp_admin_asset_acl_declared(self): + line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL') + self.assertIn('path_reg', line) + + def test_wp_admin_allowed_acl_declared(self): + line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL') + self.assertIn('path_end', line) + + def test_wp_gate_exempt_acl_declared(self): + """Scoped to the ACL line itself, not `self.cfg` as a whole -- the + surrounding prose (see hap_listener.tpl's "per-site opt-out" comment) + also spells out /etc/haproxy/wpadmin_gate_exempt.list verbatim, so an + unscoped `assertIn` would still pass with the real ACL deleted. + """ + line = require_rule(self.cfg, 'acl wp_gate_exempt', 'wp_gate_exempt ACL') + self.assertIn('/etc/haproxy/wpadmin_gate_exempt.list', line) + + def test_allowlist_entries_present(self): + """wp-login.php loads its own css/js from /wp-admin/ -- see module docstring.""" + line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL') + for entry in ALLOWLIST: + with self.subTest(entry=entry): + self.assertIn(entry, line) + + def test_static_asset_dirs_allowed(self): + line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL') + self.assertRegex(line, r'path_reg.*\(css\|js\|images\)') + + def test_install_php_is_NOT_allowlisted(self): + """install.php is deliberately gated -- a takeover vector on abandoned installs.""" + line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL') + self.assertNotIn('install.php', line) + + def test_redirect_rule_has_all_exclusions(self): + rule = require_rule_by_predicate( + self.cfg, 'http-request redirect', lambda ln: 'wp_admin_path' in ln, + 'wp-admin redirect rule') + for excl in EXCLUSIONS: + with self.subTest(exclusion=excl): + self.assertIn(excl, rule) + + def test_rule_renders_after_real_ip_resolution(self): + """is_whitelisted reads txn.real_ip; before the set-var chain it is unset.""" + _, setvar_pos = require_position(self.cfg, 'set-var(txn.real_ip)', + 'real_ip set-var chain') + _, rule_pos = require_position(self.cfg, 'wp_admin_path', 'wp_admin_path ACL/rule') + self.assertLess(setvar_pos, rule_pos, + 'wp_admin_path renders before txn.real_ip is resolved') + + def test_rule_renders_after_has_wp_logged_in_declared(self): + """HAProxy resolves ACLs as it parses; use-before-declare fails.""" + _, decl_pos = require_position(self.cfg, 'acl has_wp_logged_in', + 'has_wp_logged_in ACL declaration') + _, rule_pos = require_position(self.cfg, 'wp_admin_path', 'wp_admin_path ACL/rule') + self.assertLess(decl_pos, rule_pos, + 'wp_admin_path renders before has_wp_logged_in is declared') + + def test_only_one_has_wp_logged_in_declaration(self): + self.assertEqual(len(rule_lines(self.cfg, 'acl has_wp_logged_in')), 1) + + def test_allowlist_entries_are_anchored_to_wp_admin(self): + """Bare `path_end /admin-ajax.php` also matches + /wp-admin/evil/admin-ajax.php, which ALSO matches wp_admin_path + (path_reg only requires /wp-admin/ to appear somewhere) -- an + attacker-inserted path segment would then sail through the + allowlist ungated. Entries must be anchored to sit directly under + wp-admin/. Scoped to the captured ACL line only, since the + surrounding comment block also mentions these bare filenames. + """ + line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL') + for entry in ALLOWLIST: + with self.subTest(entry=entry): + self.assertIn('/wp-admin' + entry, line) + self.assertNotRegex( + line, r'(? 1: + raise AssertionError('%s matched more than one rule line: %r' % (what, candidates)) + return candidates[0] + + +class UriNormalisation(unittest.TestCase): + """The gate matches the RAW path; the backend normalises and decodes it. + Every gap between those is a bypass -- five were found this way. These + tests pin the normalisation that closes the gap as a class. + + NOTE: these are config-TEXT assertions. They are necessary but NOT + sufficient: the previous revision of this file passed while five live + bypasses shipped. The real evidence is the behavioural matrix run against + real haproxy 3.0.11 with raw sockets -- see + .superpowers/sdd/2026-08-14-wpadmin-edge-gate/task-4-normalize-report.md. + """ + + def setUp(self): + self.cfg = render_listener() + self.header = render_header() + + def test_experimental_directives_exposed_in_global(self): + """normalize-uri is experimental in 3.0; without this HAProxy refuses + to start (`haproxy -c` exits 1, ALERT). That does NOT crash-loop the + container, though -- see hap_header.tpl's comment and + haproxy_manager.py's start_haproxy()/do_initial_setup(): the failure + is swallowed, and the container comes up with haproxy simply never + running. This test exists so that silent-outage mode is never + reintroduced by dropping this line. + """ + lines = rule_lines(self.header, 'expose-experimental-directives') + matches = [ln for ln in lines if ln == 'expose-experimental-directives'] + self.assertTrue( + matches, 'expose-experimental-directives missing from the global section') + + def test_all_normalizers_render(self): + for norm in NORMALIZERS: + with self.subTest(normalizer=norm): + self.assertTrue( + rule_lines(self.cfg, 'normalize-uri ' + norm), + 'missing normalizer: ' + norm) + + def test_normalizer_order_decode_before_path_walkers(self): + """Reverse this order and /wp-admin/js/%2e%2e/plugins.php reaches the + ACLs as /wp-admin/js/../plugins.php -- decoded but unresolved. + """ + lines = [ln for ln in rule_lines(self.cfg, 'http-request normalize-uri') + if ln.startswith('http-request normalize-uri')] + names = [ln.split('normalize-uri ', 1)[1] for ln in lines] + self.assertEqual( + names, list(NORMALIZERS), + 'normalize-uri directives are missing, reordered, or duplicated: %r' % names) + + def test_normalisation_precedes_every_path_based_rule(self): + """A normalizer placed after a path rule normalises nothing for it.""" + norm_positions = [] + for n in NORMALIZERS: + _, pos = require_position(self.cfg, 'http-request normalize-uri ' + n, + 'normalizer: ' + n) + norm_positions.append(pos) + last_norm = max(norm_positions) + for marker in ('acl is_health_check', 'acl wp_login_path', + 'acl xmlrpc_path', 'acl wp_batch_path', + 'acl wp_admin_path', 'http-request set-path'): + with self.subTest(rule=marker): + _, marker_pos = require_position(self.cfg, marker, marker) + self.assertLess(last_norm, marker_pos, + marker + ' renders before URI normalisation') + + def test_query_sort_by_name_is_not_enabled(self): + """query-sort-by-name reorders query parameters, which would break + anything that signs or caches on the exact query string. This is NOT + because the enabled normalizers already leave the query alone -- + percent-to-uppercase and percent-decode-unreserved rewrite the WHOLE + request-target, query string included (see hap_listener.tpl's BLAST + RADIUS comment) -- it is a deliberate line between "case-fold / + decode" (no-ops under RFC 3986) and "reorder" (not a no-op for a + signed/cached query string). + """ + self.assertFalse(rule_lines(self.cfg, 'normalize-uri query-sort-by-name')) + + def test_dotdot_normalizer_uses_full(self): + """Without "full", ".." segments that climb above the root are left in + place and /../../wp-admin/plugins.php survives -- measured. + """ + lines = rule_lines(self.cfg, 'normalize-uri path-strip-dotdot') + self.assertTrue(lines, 'missing normalizer: path-strip-dotdot') + for ln in lines: + self.assertTrue(ln.endswith('path-strip-dotdot full'), ln) + + def test_encoded_separator_on_wp_admin_is_denied(self): + """percent-decode-unreserved deliberately leaves %2F encoded ("/" is + reserved), but OpenLiteSpeed decodes it and serves the file -- + /wp-admin%2Fplugins.php was measured booting PHP on the OLS tier while + matching no wp-admin ACL. Normalisation cannot close this; it needs its + own rule. + """ + denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep') + if ln.startswith('http-request deny')] + self.assertTrue(denies, 'no deny rule for encoded separators') + acl_line = require_rule(self.cfg, 'acl path_has_encoded_sep', 'path_has_encoded_sep ACL') + self.assertIn('%2f', acl_line.lower()) + + def test_encoded_separator_deny_is_scoped_to_wp_admin(self): + """A blanket "deny any %2F in any path" would break non-WordPress + customer apps that legitimately pass an encoded slash in a path + parameter. The deny must be conditioned on the path mentioning + wp-admin. + """ + denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep') + if ln.startswith('http-request deny')] + self.assertTrue(denies, 'no deny rule for encoded separators') + for d in denies: + self.assertIn('wp_admin_word', d) + + def test_encoded_separator_deny_honors_the_same_whitelist(self): + denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep') + if ln.startswith('http-request deny')] + self.assertTrue(denies, 'no deny rule for encoded separators') + for d in denies: + for excl in ('!has_wp_logged_in', '!wp_gate_exempt', '!is_local', + '!is_trusted_ip', '!is_whitelisted'): + with self.subTest(rule=d, exclusion=excl): + self.assertIn(excl, d) + + def test_encoded_separator_acl_matches_a_substring_not_a_prefix(self): + """/blog%2Fwp-admin/plugins.php hides the separator BEFORE "wp-admin", + where an anchored pattern never matches, and OLS still resolves it. + """ + acl_line = require_rule(self.cfg, 'acl path_has_encoded_sep', 'path_has_encoded_sep ACL') + self.assertIn('-m sub', acl_line) + + def test_wp_admin_asset_bypass_cannot_cover_a_php_entrypoint(self): + """The asset bypass anchored its prefix but not its suffix, so + /wp-admin/css/../plugins.php took it and the backend then resolved + ".." and booted plugins.php. path-strip-dotdot is the real fix; this + keeps the bypass structurally incapable of covering PHP. + """ + acl_line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL') + self.assertIn('.php', acl_line, + 'wp_admin_asset must exclude .php explicitly') + + def test_case_insensitive_acl_and_regsub_are_kept_in_sync(self): + """A case-insensitive wp_admin_path with a case-sensitive regsub is an + INFINITE REDIRECT LOOP: regsub finds no "/wp-admin/" in + "/WP-ADMIN/plugins.php", returns `path` unchanged, and the Location + then points at the request's own URL. + """ + acl_line = require_rule(self.cfg, 'acl wp_admin_path', 'wp_admin_path ACL') + setvar_line = require_rule(self.cfg, 'set-var(txn.wp_login_url)', 'wp_login_url set-var') + acl_ci = bool(re.search(r'path_reg\s+-i\s', acl_line)) + regsub_ci = bool(re.search(r'regsub\([^)]*,\s*i\)', setvar_line)) + self.assertEqual( + acl_ci, regsub_ci, + 'wp_admin_path case-sensitivity (%s) and regsub flags (%s) disagree' + % (acl_line, setvar_line)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/validate-rendered-config.py b/scripts/validate-rendered-config.py new file mode 100644 index 0000000..539789b --- /dev/null +++ b/scripts/validate-rendered-config.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Build gate: render the real HAProxy config and hand it to the real `haproxy -c`. + +Why this file exists +-------------------- +On 2026-08-14 a change to hap_listener.tpl rendered perfectly, passed every +unit test in scripts/ (13 green), and was then rejected outright by HAProxy: + + [ALERT] config : parsing [/etc/haproxy/haproxy.cfg:...] : + invalid arg 2 in converter 'regsub' : ... unexpected empty + replacement string + +Nothing between "commit" and "running in production" would have caught it. +The unit tests assert on the *text* of the rendered config with regexes, which +tells you what the template says, never whether HAProxy will accept it. And +scripts/test-config-rollback.py stubs the `haproxy` binary with a shell script +that only rejects a literal sentinel token, so its "validation" has never +parsed a single line of real HAProxy syntax. + +The failure mode this guards is not cosmetic. When haproxy.cfg is invalid, +scripts/init.py refuses to start HAProxy but the container still comes up: +ports 80/443 are unbound, every site on the host is down, and /health keeps +answering 200 because the Flask API is fine. + +So: render the config through the SAME code path production uses +(haproxy_manager.generate_config(), templates and all), then run the actual +`haproxy -c` against the result and gate on its exit code. + +This runs as a RUN step in the Dockerfile, which means it also validates +against the exact haproxy binary that ships in the image being built - note +that the Dockerfile installs haproxy UNPINNED, so that binary can move under +us between builds. Any syntax the new binary rejects now fails the build +instead of failing at 3am on an edge node. + +What it covers +-------------- + * every template generate_config() touches, assembled in the real order + * a domain with SSL + a backend, a wildcard domain, a cert-only domain with + no backend, and two template_override backends + * blocked-IP map entries (single IP and CIDR) + * BOTH sides of the two conditional blocks in hap_listener.tpl - + {%- if suspension_enabled %} and {%- if coraza_spoe_backend %} - because a + syntax error inside a conditional ships undetected otherwise. Scenario + "full" turns both on; scenario "default" leaves both off, which is the + byte-identical-to-standalone shape. + +Warnings vs failures +-------------------- +`haproxy -c` emits warnings on a clean config here (at minimum "Can't load +stats file" because /var/lib/haproxy/stats.dat doesn't exist at build time, +plus assorted path_reg/ACL advisories). Those are NOT failures. This gate keys +on the process EXIT CODE only, and dumps the full output when it is non-zero. + +Running +------- + python3 scripts/validate-rendered-config.py + +Needs the real `haproxy` binary, the application's Python dependencies, and +write access to /etc/haproxy (several templates reference files there by +absolute path - see _REAL_PATH_NOTE below). Inside the image build all three +hold. On a workstation, run it in the container instead. +""" + +import logging +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile + +MODULE_DIR = os.path.abspath( + os.environ.get('HAPROXY_MANAGER_DIR', + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +) +os.chdir(MODULE_DIR) +sys.path.insert(0, MODULE_DIR) + +# Same trick the other suites use: haproxy_manager configures logging at import +# time against /var/log/haproxy-manager.log. Redirect the handlers so this runs +# without root and without polluting the image's log files. +_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-validate-logs-') +_real_file_handler = logging.FileHandler +logging.FileHandler = ( + lambda filename, *a, **kw: _real_file_handler( + os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw) +) +try: + import haproxy_manager as hm # noqa: E402 +finally: + logging.FileHandler = _real_file_handler + +# The application logs a lot at INFO during a render, and legitimately logs at +# ERROR about things that are only true in this harness ("no existing HAProxy +# config on disk ... ROLLBACK IS NOT AVAILABLE" - correct, there is no live +# config during a build). Silence it so the build log carries the gate's own +# verdict and haproxy's output, which is what matters. +logging.getLogger('haproxy_manager').setLevel(logging.CRITICAL) + + +# _REAL_PATH_NOTE +# --------------- +# Most paths haproxy_manager writes to are module-level constants and are +# redirected into a temp dir below. Two cannot be: +# +# /etc/haproxy/blocked_ips.map - hardcoded inside hap_listener.tpl's +# map_ip() converter +# /etc/haproxy/coraza-spoe.cfg - hardcoded in the `filter spoe engine` +# line, and parsed by haproxy -c +# +# Redirecting the constants without editing the templates would just make +# haproxy read a different (missing) file, so those two are left at their real +# paths. Everything this script creates under /etc/haproxy is removed again on +# exit; pre-existing files (the baked trusted_ips.*) are never touched. +ETC_HAPROXY = '/etc/haproxy' + +# Files referenced with `-f` / map_ip() from the templates. A missing `-f` file +# is a FATAL haproxy error, so a gate that didn't create these would fail for +# reasons that have nothing to do with the config being tested. +STUB_FILES = { + os.path.join(ETC_HAPROXY, 'trusted_ips.list'): '# validation stub\n203.0.113.10\n', + os.path.join(ETC_HAPROXY, 'trusted_ips.map'): '# validation stub\n203.0.113.11 1\n', + os.path.join(ETC_HAPROXY, 'cloudflare_ips.list'): '# validation stub\n198.51.100.0/24\n', + os.path.join(ETC_HAPROXY, 'trusted_proxies.list'): '# validation stub\n192.0.2.0/24\n', + os.path.join(ETC_HAPROXY, 'wpadmin_gate_exempt.list'): '# validation stub\nexempt.example.test\n', + os.path.join(ETC_HAPROXY, 'suspended_domains.list'): 'suspended.example.test\n', + # `lf-file` on the Coraza deny rule; loaded at parse time. Present in the + # image (COPY errors /haproxy/errors), stubbed for anything else. + '/haproxy/errors/403-waf.html': 'blocked %[unique-id]\n', +} + +# (suspension_enabled, coraza_spoe_backend) combinations to render + validate. +SCENARIOS = ( + ('default', {}), + ('full', { + 'HAPROXY_SUSPENSION_ENABLED': 'true', + 'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000', + }), +) + + +def log(msg): + sys.stdout.write(f'[validate-config] {msg}\n') + sys.stdout.flush() + + +def fail(msg): + sys.stderr.write(f'[validate-config] FAIL: {msg}\n') + sys.stderr.flush() + raise SystemExit(1) + + +class CreatedFiles: + """Tracks what we put on disk outside the temp dir so it can be removed. + + Two sources: files we create explicitly, and files generate_config() itself + writes into /etc/haproxy (blocked_ips.map, coraza-spoe.cfg and their + .backup copies). The latter are caught by diffing the directory listing, + which also picks up anything a future change starts writing there. + """ + + def __init__(self): + self.explicit = [] + self.etc_before = self._listdir(ETC_HAPROXY) + + @staticmethod + def _listdir(path): + try: + return set(os.listdir(path)) + except OSError: + return set() + + def ensure(self, path, content): + """Create path with content if it does not already exist.""" + if os.path.exists(path): + return + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as fh: + fh.write(content) + os.chmod(path, 0o644) + self.explicit.append(path) + + def cleanup(self): + for path in self.explicit: + try: + os.unlink(path) + except OSError: + pass + for name in self._listdir(ETC_HAPROXY) - self.etc_before: + try: + os.unlink(os.path.join(ETC_HAPROXY, name)) + except OSError: + pass + + +def require_haproxy_binary(): + """Fail closed. A gate that skips itself when the binary is missing is a + gate that would have let the 2026-08-14 change through.""" + path = shutil.which('haproxy') + if not path: + fail('no `haproxy` binary on PATH - this gate cannot validate anything. ' + 'Run it inside the image (the Dockerfile installs haproxy).') + version = subprocess.run([path, '-v'], capture_output=True, text=True) + log(f'using {path}: {version.stdout.strip().splitlines()[0] if version.stdout else "unknown version"}') + + +def make_self_signed_cert(certs_dir): + """HAProxy loads every file in the `bind ... ssl crt ` directory at + parse time, so the directory has to hold a real, loadable bundle.""" + os.makedirs(certs_dir, exist_ok=True) + cert = os.path.join(certs_dir, 'cert.tmp') + key = os.path.join(certs_dir, 'key.tmp') + subprocess.run( + ['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', + '-keyout', key, '-out', cert, '-days', '1', + '-subj', '/CN=validate.example.test'], + check=True, capture_output=True, + ) + bundle = os.path.join(certs_dir, 'validate.example.test.pem') + with open(bundle, 'w') as out: + for part in (cert, key): + with open(part) as fh: + out.write(fh.read()) + os.unlink(cert) + os.unlink(key) + os.chmod(bundle, 0o600) + + +def seed_database(db_path): + """A representative fleet: SSL + backend, wildcard, cert-only, overrides.""" + hm.DB_FILE = db_path + hm.init_db() + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + + def add_site(domain, backend, ssl_enabled=1, wildcard=0, override=None, + servers=(('web1', '10.0.0.10', 8080, 'check'),)): + cur.execute( + 'INSERT INTO domains (domain, ssl_enabled, ssl_cert_path, ' + 'template_override, is_wildcard) VALUES (?, ?, ?, ?, ?)', + (domain, ssl_enabled, f'/etc/haproxy/certs/{domain}.pem', + override, wildcard), + ) + domain_id = cur.lastrowid + cur.execute('INSERT INTO backends (name, domain_id, settings) ' + 'VALUES (?, ?, ?)', (backend, domain_id, None)) + backend_id = cur.lastrowid + for name, addr, port, opts in servers: + cur.execute( + 'INSERT INTO backend_servers (backend_id, server_name, ' + 'server_address, server_port, server_options) ' + 'VALUES (?, ?, ?, ?, ?)', + (backend_id, name, addr, port, opts), + ) + + add_site('site-one.example.test', 'site-one', + servers=(('web1', '10.0.0.10', 8080, 'check'), + ('web2', '10.0.0.11', 8080, 'check backup'))) + add_site('site-two.example.test', 'site-two', ssl_enabled=0) + add_site('*.wildcard.example.test', 'wildcard-site', wildcard=1) + add_site('ws.example.test', 'ws-site', override='hap_backend_websocket') + add_site('sse.example.test', 'sse-site', override='hap_backend_longlived') + + # Cert/management-only domain: registered for certificates, no backend. + # generate_config() has an explicit branch for this. + cur.execute( + 'INSERT INTO domains (domain, ssl_enabled, ssl_cert_path, ' + 'template_override, is_wildcard) VALUES (?, ?, ?, ?, ?)', + ('panel.example.test', 1, '/etc/haproxy/certs/panel.pem', None, 0), + ) + + # Both map_ip() shapes: a single address and a CIDR. + for ip in ('203.0.113.66', '198.51.100.0/24'): + cur.execute('INSERT INTO blocked_ips (ip_address, reason, blocked_by) ' + 'VALUES (?, ?, ?)', (ip, 'validation fixture', 'gate')) + conn.commit() + + +def render(scenario_name, env_overrides, workdir): + """Render via generate_config() and return the path to the assembled config. + + generate_config() is the real production entry point: it takes the rollback + snapshot, writes the blocked-IP map, renders every template, and writes + haproxy.cfg. Only the reload is stubbed - there is no HAProxy process to + reload during a build, and validating the file is the whole point. + """ + scenario_dir = os.path.join(workdir, scenario_name) + certs_dir = os.path.join(scenario_dir, 'certs') + os.makedirs(scenario_dir) + make_self_signed_cert(certs_dir) + + hm.HAPROXY_CONFIG_PATH = os.path.join(scenario_dir, 'haproxy.cfg') + hm.HAPROXY_BACKUP_PATH = os.path.join(scenario_dir, 'haproxy.cfg.backup') + hm.CLUSTER_SECRET_PATH = os.path.join(scenario_dir, 'cluster-secret') + hm.HAPROXY_SOCKET_PATH = os.path.join(scenario_dir, 'haproxy.sock') + hm.SSL_CERTS_DIR = certs_dir + seed_database(os.path.join(scenario_dir, 'haproxy_config.db')) + + saved_env = {} + for key in ('HAPROXY_SUSPENSION_ENABLED', 'HAPROXY_CORAZA_SPOE_BACKEND'): + saved_env[key] = os.environ.pop(key, None) + os.environ.update(env_overrides) + + real_reload = hm.reload_haproxy_safely + hm.reload_haproxy_safely = lambda *a, **kw: (True, 'reload skipped: build-time validation') + try: + hm.generate_config() + finally: + hm.reload_haproxy_safely = real_reload + for key, value in saved_env.items(): + os.environ.pop(key, None) + if value is not None: + os.environ[key] = value + + return hm.HAPROXY_CONFIG_PATH + + +def assert_scenario_branches(scenario_name, config_path, env_overrides): + """Cheap sanity check that the conditional blocks actually rendered. + + Without this, a template refactor that silently stopped emitting the + suspension or Coraza block would leave the gate passing while covering + less than it claims to. + """ + with open(config_path) as fh: + text = fh.read() + expected = { + 'suspension': ('acl is_suspended_domain', + 'HAPROXY_SUSPENSION_ENABLED' in env_overrides), + 'coraza': ('filter spoe engine coraza', + 'HAPROXY_CORAZA_SPOE_BACKEND' in env_overrides), + } + for label, (needle, should_be_present) in expected.items(): + present = needle in text + if present != should_be_present: + fail(f'[{scenario_name}] {label} block {"missing" if should_be_present else "unexpectedly present"} ' + f'in the rendered config (looked for {needle!r}). The gate is ' + f'not covering what it thinks it is.') + + +def _dump_context(config_path, output): + """Print the rendered lines HAProxy complained about. + + The temp dir is deleted on the way out, so the build log has to carry the + evidence. HAProxy reports `parsing [:]`; show a window around + each reported line rather than dumping ~1500 lines of config. + """ + line_numbers = sorted({ + int(n) for n in re.findall( + r'parsing \[' + re.escape(config_path) + r':(\d+)\]', output) + }) + if not line_numbers: + return + with open(config_path) as fh: + lines = fh.read().splitlines() + sys.stderr.write('[validate-config] --- rendered config around the error ---\n') + for number in line_numbers: + start = max(1, number - 6) + end = min(len(lines), number + 4) + for index in range(start, end + 1): + marker = '>>' if index == number else ' ' + sys.stderr.write(f'{marker}{index:6d}| {lines[index - 1]}\n') + sys.stderr.write('[validate-config] ---\n') + + +def validate(scenario_name, config_path): + result = subprocess.run(['haproxy', '-c', '-f', config_path], + capture_output=True, text=True) + if result.returncode != 0: + output = result.stdout + result.stderr + sys.stderr.write( + f'\n[validate-config] ===== {scenario_name}: haproxy REJECTED the ' + f'rendered configuration (exit {result.returncode}) =====\n') + sys.stderr.write(output if output.endswith('\n') else output + '\n') + _dump_context(config_path, output) + sys.stderr.write( + '[validate-config] This is a real HAProxy parse failure. Shipping it ' + 'would leave the container Up with ports 80/443 unbound and every ' + 'site on the host down, while /health still returns 200.\n') + sys.stderr.flush() + raise SystemExit(1) + + # Exit code 0 is the verdict. Warnings are expected and are NOT failures: + # "Can't load stats file" always fires at build time, and HAProxy emits + # path_reg/ACL advisories on a perfectly valid config. + noise = (result.stdout + result.stderr).strip() + log(f'{scenario_name}: haproxy -c OK (exit 0)') + if noise: + for line in noise.splitlines(): + log(f' {scenario_name}: haproxy said: {line}') + + +def main(): + require_haproxy_binary() + + if not os.path.isdir(ETC_HAPROXY) or not os.access(ETC_HAPROXY, os.W_OK): + fail(f'{ETC_HAPROXY} must exist and be writable - several templates ' + 'reference files there by absolute path. Run this inside the image.') + + created = CreatedFiles() + workdir = tempfile.mkdtemp(prefix='haproxy-validate-') + try: + for path, content in STUB_FILES.items(): + created.ensure(path, content) + + for scenario_name, env_overrides in SCENARIOS: + log(f'rendering scenario "{scenario_name}" ' + f'({env_overrides or "no optional features"})') + config_path = render(scenario_name, env_overrides, workdir) + assert_scenario_branches(scenario_name, config_path, env_overrides) + validate(scenario_name, config_path) + finally: + created.cleanup() + shutil.rmtree(workdir, ignore_errors=True) + shutil.rmtree(_LOG_DIR, ignore_errors=True) + + log('all scenarios accepted by the real haproxy binary') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/templates/hap_header.tpl b/templates/hap_header.tpl index 322ae0e..35b7773 100644 --- a/templates/hap_header.tpl +++ b/templates/hap_header.tpl @@ -27,6 +27,36 @@ global # SSL and Performance tune.ssl.default-dh-param 2048 + # Required by the `http-request normalize-uri` chain at the top of the + # `web` frontend (hap_listener.tpl). normalize-uri is still flagged + # EXPERIMENTAL in HAProxy 3.0, and HAProxy REFUSES TO START without this + # opt-in -- not a warning, a fatal: + # [ALERT] config : parsing [...] : 'normalize-uri' action is + # experimental, must be allowed via a global + # 'expose-experimental-directives' + # (verified against real haproxy 3.0.11-9e587df: `haproxy -c` exits 1). + # So this line and the normalize-uri rules must be added/removed together. + # + # Dropping this one alone does NOT crash-loop the container -- the truth + # is worse: it is a SILENT TOTAL OUTAGE that nothing escalates. Container + # init (scripts/init.py -> haproxy_manager.do_initial_setup()) calls + # generate_config() (which still succeeds -- Jinja doesn't validate + # HAProxy semantics) and then start_haproxy(), which runs `haproxy -c`, + # sees it fail, logs an error, and RETURNS WITHOUT RAISING. init.py exits + # 0. scripts/start-up.sh then execs gunicorn as PID 1 regardless. Result: + # the container stays "Up", ports 80/443 are never bound, EVERY SITE ON + # THE HOST IS DOWN, and the in-container supervisor loop + # (ensure_haproxy.py, every HAPROXY_SUPERVISOR_INTERVAL seconds) retries + # the identical failing render forever without ever escalating. Worse + # still, GET /health keeps returning HTTP 200 -- health_check() only + # answers 500 on a database error; a dead haproxy just flips the JSON + # body's "haproxy_status" to "stopped" while the status code a naive + # monitor checks never changes. Do not trust /health alone to catch this. + # + # This exposes ONLY the experimental directives that are actually used -- + # it does not change the behaviour of anything else in this file. + expose-experimental-directives + # HTTP/3 over QUIC. The Debian haproxy package is built against system # OpenSSL via the compatibility shim (USE_QUIC_OPENSSL_COMPAT), which is # not a native QUIC TLS stack. HAProxy therefore rejects `quic*@` binds diff --git a/templates/hap_listener.tpl b/templates/hap_listener.tpl index 4ed87ae..463afff 100644 --- a/templates/hap_listener.tpl +++ b/templates/hap_listener.tpl @@ -22,6 +22,103 @@ frontend web # Capture Host header so it appears in httplog output (in %hr field) http-request capture req.hdr(Host) len 64 + # --- URI normalisation (MUST be the first path-touching block here) --- + # Every path-based control in this frontend (the ACME health-check bypass, + # wp-login, xmlrpc, the wp-json/batch virtual patch, the wp-admin gate, the + # blocked-IP and suspension set-path rules, and everything Coraza inspects) + # matched the RAW request-target while the backend NORMALISED and DECODED + # it before resolving a file. Every gap between those two behaviours is a + # bypass, and each one had to be patched individually. Five were found in + # the wp-admin gate alone, all the same class: + # + # //wp-admin/plugins.php raw path starts "//" -> the safe-path + # guard failed -> request fell through + # ungated, backend served it + # /wp-admin/css/../plugins.php matched the css/js/images asset + # bypass; backend resolved ".." and + # booted plugins.php + # /wp-admin/js/%2e%2e/plugins.php same, with the ".." percent-encoded + # /wp%2Dadmin/plugins.php "wp-admin" spelled with %2D never + # matched any wp-admin ACL at all + # /wp-admin%2Fplugins.php encoded separator; see the dedicated + # rule in the wp-admin gate below + # + # Rather than keep bolting a counter-pattern onto each rule, normalise the + # URI once, here, so every rule below matches the SAME string the backend + # will resolve. HAProxy rewrites the request-target in place, so the + # backend receives the normalised form too. + # + # ORDER IS LOAD-BEARING and was determined empirically against real + # haproxy 3.0.11, not from the docs. The decoders must run BEFORE the path + # walkers: with the reverse order, /wp-admin/js/%2e%2e/plugins.php ends up + # as /wp-admin/js/../plugins.php -- decoded, but the ".." left unresolved, + # because path-strip-dotdot had already run by the time the "%2e%2e" + # became "..". Verified both directions side by side. + # + # percent-to-uppercase %2f -> %2F. Canonicalises the spelling of + # whatever stays encoded, so downstream rules + # need one case of each escape, not two. + # percent-decode-unreserved Decodes ONLY RFC 3986 unreserved chars + # (A-Za-z0-9-._~). This is what turns %2e%2e + # into .. and wp%2Dadmin into wp-admin. + # Reserved escapes are deliberately left + # alone -- %2F in particular, which is why + # the wp-admin gate needs its own encoded- + # slash rule (see below). + # path-merge-slashes //x -> /x. Also removes the entire class of + # "leading // defeats an anchored regex". + # path-strip-dot /a/./b -> /a/b. + # path-strip-dotdot full /a/b/../c -> /a/c. "full" additionally + # resolves ".." segments that would climb + # above the root (/../../wp-admin/x.php -> + # /wp-admin/x.php); without "full" HAProxy + # leaves those in place and the vector + # survives -- measured, both forms tested. + # + # DELIBERATELY NOT ENABLED: query-sort-by-name. Reordering query-string + # parameters would silently break anything that signs or caches on the + # exact query string (signed asset URLs, HMAC'd callbacks, CDN cache + # keys). This is NOT because the enabled normalizers already leave the + # query alone -- see BLAST RADIUS just below, they don't -- it is a + # deliberate line drawn between "case-fold / decode", which RFC 3986 + # defines as no-ops, and "reorder", which is not a no-op for a caller + # treating the query as an opaque signed string. + # + # BLAST RADIUS: this block applies to EVERY request for EVERY site on + # EVERY tier, so the decoding was kept minimal on purpose -- and it is + # NOT scoped to the path. percent-to-uppercase and percent-decode- + # unreserved normalise the WHOLE request-target as HAProxy parses it -- + # query string included, not just the path component the ACLs below + # match on -- and the BACKEND receives the rewritten query on the wire, + # not just an internal haproxy view of it. Measured against real HAProxy + # 3.0.11: + # /a?sig=%2babc%2fdef -> /a?sig=%2Babc%2Fdef (percent-hex upper-cased) + # /a?b=%41%42%43 -> /a?b=ABC (unreserved chars decoded) + # /a?tok=%7e%2d%5f%2e -> /a?tok=~-_. (unreserved chars decoded) + # Only parameter ORDER is preserved -- that guarantee is exactly why + # query-sort-by-name above is the one normalizer in this family left + # disabled. Per RFC 3986 both enabled rewrites are defined as the same + # URI (case in a percent-escape, and an unreserved character vs. its + # escape, carry no distinct meaning), but "the same URI" is not "the + # same bytes": an application that HMACs or otherwise signs the RAW + # query string, rather than parsing it first, could see a mutated value + # and fail to verify an otherwise-legitimate request. Checked against + # this fleet: no .NET backends (.NET's UrlEncode emits lowercase + # percent-hex, which percent-to-uppercase would rewrite) and no + # URL-in-path proxies, so there is no known victim today -- but do not + # assume "path only" from this block; that was the actual bug in an + # earlier draft of this comment. + # + # normalize-uri is EXPERIMENTAL in 3.0 and requires + # `expose-experimental-directives` in the global section + # (hap_header.tpl). Without it HAProxy does not start. Remove one and you + # must remove the other. + http-request normalize-uri percent-to-uppercase + http-request normalize-uri percent-decode-unreserved + http-request normalize-uri path-merge-slashes + http-request normalize-uri path-strip-dot + http-request normalize-uri path-strip-dotdot full + # --- Trusted-proxy gate (MUST precede real-IP resolution below) --- # CF-Connecting-IP / X-Real-IP / X-Forwarded-For are client-supplied. Any # peer that is not a known reverse proxy gets them stripped, so the @@ -183,6 +280,248 @@ frontend web http-request deny deny_status 403 if wp_batch_route !has_wp_logged_in !is_local !is_trusted_ip !is_whitelisted http-request deny deny_status 403 if wp_batch_route_enc !has_wp_logged_in !is_local !is_trusted_ip !is_whitelisted + # --- WordPress admin edge gate --- + # Measured on whp02, 2026-08-14: distributed unauthenticated GETs booting + # WordPress just to bounce back a login redirect -- + # 2801 GET /wp-admin/profile.php 2781 GET /wp-admin/edit.php 2761 GET /wp-admin/plugins.php + # -- spread across many source IPs at roughly 4 req/min per IP, each hit + # burning a PHP-FPM/lsphp worker. Two sites absorbed 1,243 resulting 503s + # in nine hours as their pools saturated. + # + # IDENTITY, NOT RATE. This is the one rule in this file that tracks + # nothing and has no stick-table counter or threshold. Every rate-based + # control above (the generic limits, wp_bruteforce, xmlrpc_bruteforce) is + # per-IP, and per-IP rate is exactly what this attack is engineered to + # stay under: ~4 req/min from any single IP is indistinguishable from a + # slow human, and the source set is large enough that no threshold can be + # lowered to catch it without also catching real visitors. There is also + # no free stick-table slot left to try anyway -- sc0/sc1/sc2 are already + # used above and HAProxy's default tune.stick-counters is 3, so a fourth + # tracked counter is not an option here. DO NOT "simplify" this into a + # rate/threshold rule later: the whole point is that a threshold cannot + # see this traffic. Instead we gate on identity -- a real logged-in + # WordPress user always carries a wordpress_logged_in_* cookie (the same + # ACL the wp2shell block above already declares), and an unauthenticated + # request to a wp-admin page has no legitimate reason to boot PHP at all. + # + # 302, not 403. WordPress itself redirects an unauthenticated /wp-admin/ + # request to wp-login.php, so replicating that at the edge means an admin + # whose session merely expired lands on the normal login screen instead + # of an error page -- we are not trading a bot problem for a support + # ticket. Bots get a cheap redirect they ignore. + # + # path_reg, not path_beg. A subdirectory install at /blog/wp-admin/ slips + # past a prefix match; path_reg with an optional leading "/" catches both + # root and subdirectory installs, same reasoning as the wp-login and + # xmlrpc path_end rules above. + # + # regsub rewrites the redirect target itself, so /blog/wp-admin/x.php + # redirects to /blog/wp-login.php rather than 404ing at the site root. + # + # DO NOT write regsub's regex argument with a capturing group / literal + # parentheses, e.g. regsub((^|/)wp-admin/.*,\1wp-login.php) -- neither + # inlined into the redirect's `location` nor in a standalone set-var. + # HAProxy 3.0.11's converter-argument parser counts parens to find the + # end of the regsub(...) call itself, so the *inner* "(^|/)" grouping + # parens are misread as closing the outer call early -- it does not + # matter whether the argument is quoted ("...": still fails) or the + # parens are backslash-escaped (\(...\): still fails). Every such form + # was verified against real HAProxy 3.0.11-1+deb13u3 and all produce the + # same ALERT: "invalid arg 2 in converter 'regsub' : missing arguments + # (got 1/2)". This is a converter-argument-parsing limitation, not a + # log-format/`%[...]` issue -- the identical failure reproduces in a + # plain set-var (outside any log-format string), which rules out the + # `location` value's log-format context as the cause. + # + # The fix sidesteps groups/backreferences entirely: HTTP paths always + # start with "/", so the leading "(^|/)" alternation is redundant -- + # matching the literal substring "/wp-admin/" (both slashes, no group) + # is sufficient to anchor to a real path segment (a false match like + # "/somewp-admin/" doesn't contain "/wp-admin/" as a substring, since + # there's no "/" directly before "wp-admin"). No backreference is + # needed either: regsub only replaces the matched substring, so + # replacing "/wp-admin/.*" with a literal "/wp-login.php" leaves + # whatever precedes it (the subdirectory-install prefix, if any) + # untouched. Computed in its own set-var so it is a plain sample + # expression, not something baked into the redirect's log-format + # string. Behaviorally verified live against real HAProxy 3.0.11: + # /wp-admin/edit.php -> /wp-login.php and + # /blog/wp-admin/plugins.php -> /blog/wp-login.php, both with + # redirect_to preserved. See + # .superpowers/sdd/2026-08-14-wpadmin-edge-gate/task-3b-report.md. + # + # redirect_to carries the path only (%[path,url_enc]), not the query + # string -- deliberate, see design spec. An admin bounced off + # post.php?post=123&action=edit lands back on a blank post.php rather + # than that exact post. Capturing the full URI needs capture.req.uri + # (extra config, and the captured value is length-capped) for a benefit + # that only matters on session expiry, so it was not worth it. + # + # THE ALLOWLIST IS MEASURED, NOT GUESSED -- taken from actual fleet + # traffic returning 200 on /wp-admin/*. admin-ajax.php and admin-post.php + # are the standard front-end AJAX/form-handler endpoints real themes and + # plugins call while logged out. Critically, wp-login.php loads its OWN + # css/js FROM /wp-admin/ (load-styles.php, load-scripts.php, and the + # static wp_admin_asset dirs below) -- miss those and every login page on + # the fleet renders unstyled with a broken password-strength meter, while + # wp-login.php itself still returns 200, making it a silent regression + # that "looks like" the gate is working. + # + # Each allowlist entry is anchored to /wp-admin/, not a bare + # filename suffix. A bare `path_end /admin-ajax.php` also matches + # /wp-admin/evil/admin-ajax.php -- which ALSO matches wp_admin_path + # (path_reg only requires /wp-admin/ to appear somewhere), so an + # attacker-inserted path segment would sail through this allowlist + # ungated and boot full WordPress, exactly the resource exhaustion this + # gate exists to stop. Anchoring still covers subdirectory installs via + # suffix matching (/blog/wp-admin/admin-ajax.php ends with + # /wp-admin/admin-ajax.php) while rejecting an inserted directory. + # + # install.php is DELIBERATELY NOT allowlisted. It is legitimately + # reachable without a cookie during a fresh install, but it is also a + # standing scanner target and a real takeover vector on a site that was + # half-installed and then abandoned. Anyone genuinely installing uses the + # per-site exempt-list opt-out below instead. + # + # Honors the same whitelist as every other rule in this frontend + # (RFC1918 / trusted_ips.list / trusted_ips.map), and a per-site opt-out + # via /etc/haproxy/wpadmin_gate_exempt.list (operator-managed, seeded + # empty by start-up.sh) for sites where a plugin legitimately serves + # unauthenticated visitors from a /wp-admin/ URL outside this allowlist. + # EVERY ACL BELOW MATCHES THE NORMALISED PATH. The normalize-uri chain at + # the top of this frontend has already merged duplicate slashes, resolved + # "." / ".." segments (including percent-encoded ones) and decoded + # unreserved escapes by the time these run, so these patterns only have to + # describe the ONE canonical spelling the backend will resolve -- they do + # not have to anticipate every encoding of it. That is the whole point of + # the normalisation block; do not "harden" these regexes by re-adding + # encoding variants, fix the normalisation instead. + # + # wp_admin_safe_path guards against an OPEN REDIRECT this gate would + # otherwise introduce. The redirect target below is built by rewriting + # `path` with regsub -- regsub only replaces the matched substring, so + # everything BEFORE the matched "/wp-admin/" survives untouched in the + # output. `path` is not guaranteed to be a clean site-relative string; + # three concrete requests turn that survival into an off-site + # `Location:` header: + # //evil.example.com/wp-admin/x.php -> //evil.example.com/wp-login.php + # (protocol-relative -- browsers resolve "//host/path" to + # "https://host/path", so this redirects off-site with no scheme + # needed). NOW NEUTRALISED UPSTREAM: path-merge-slashes rewrites this + # to /evil.example.com/wp-admin/x.php before any ACL sees it, so the + # Location becomes the same-origin /evil.example.com/wp-login.php. + # Verified live. + # /\evil.example.com/wp-admin/x.php -> /\evil.example.com/wp-login.php + # (browsers normalise a leading "/\" the same as "//"). STILL LIVE + # after normalisation -- a backslash is not a slash, so no normalizer + # touches it. This ACL is the only thing that stops it. + # https://evil.example.com/wp-admin/x.php -> https://evil.example.com/wp-login.php + # (RFC 7230 absolute-form request targets can make HAProxy's `path` + # fetch return a full URI, not just the path component). HAProxy's own + # H1 parser answers 400 on this frontend; this ACL is the backstop. + # So wp_admin_safe_path is NOT redundant with the normalisation and must + # not be deleted as such -- one of its three vectors survives normalisation + # untouched. + # + # It is used TWO ways, and the pair matters: + # - as a POSITIVE condition on the redirect, so a pathological path can + # never produce a `Location:` header at all; and + # - as an explicit deny, so such a path is not merely un-redirected. + # The deny is what closes the failure mode the positive-condition form + # introduced on its own: "not redirected" used to mean "falls through to + # the backend UNGATED", i.e. the exact PHP-booting request this gate + # exists to stop, reachable by prefixing "//" (that specific spelling is + # now normalised away, but "/\" is not). Post-normalisation the only + # paths that reach the deny are "/\..." ones, which cannot resolve to a + # real file on any tier, so nothing legitimate is denied. + # -i (case-insensitive) and the matching ",i" flag on the regsub below are + # a PAIR -- adding one without the other produces an infinite redirect + # loop, because a case-sensitive regsub finds no "/wp-admin/" in + # "/WP-ADMIN/plugins.php", returns `path` UNCHANGED, and the Location then + # points at the request's own URL. Verified live that the pair is correct: + # /WP-ADMIN/plugins.php -> /wp-login.php and /blog/WP-Admin/plugins.php -> + # /blog/wp-login.php. + # + # On this fleet's Linux backends /WP-ADMIN/plugins.php 404s without booting + # PHP, so this is hardening rather than a live-bypass fix; it matters if a + # docroot ever sits on a case-insensitive mount, where that same request + # WOULD boot PHP. The cost is that a site with a real directory literally + # named e.g. /docs/WP-Admin/ now gets gated -- the same false positive the + # lowercase pattern already has, which is what the per-site exempt list + # exists to resolve. + acl wp_admin_path path_reg -i (^|/)wp-admin/ + # Four literal backslashes here is NOT a typo. HAProxy's config-line word + # parser treats backslash as its OWN escape character before the value + # ever reaches the regex engine: "\\" (two backslashes) in the config + # collapses to one literal backslash by the time PCRE compiles it, which + # leaves an unterminated character class ("[^/\]") and fails with + # "missing terminating ] for character class" -- verified against real + # HAProxy 3.0.11. Four backslashes ("\\\\") collapse to two ("\\"), + # which PCRE then reads as a single escaped-backslash class member -- + # the intended "reject a literal backslash" semantics. + acl wp_admin_safe_path path_reg ^/[^/\\\\] + acl wp_admin_allowed path_end /wp-admin/admin-ajax.php /wp-admin/admin-post.php /wp-admin/load-styles.php /wp-admin/load-scripts.php + # The (?!.*\.php) lookahead is DEFENCE IN DEPTH, not the primary fix. This + # ACL grants an un-gated bypass to everything under wp-admin/css|js|images, + # and it used to anchor its prefix but not its suffix, so + # /wp-admin/css/../plugins.php took the bypass and the backend then + # resolved ".." and booted plugins.php. path-strip-dotdot now rewrites that + # to /wp-admin/plugins.php before this ACL runs, which is the real fix; the + # lookahead additionally makes the bypass structurally incapable of + # covering a PHP entrypoint even if a future encoding trick survives + # normalisation. It excludes ".php" ONLY -- no static asset contains that + # substring, so it cannot cause the silent "login page renders unstyled" + # regression that an extension allowlist would risk. Requires PCRE2, which + # both the Debian (deployed) and Alpine haproxy builds have (+PCRE2). + acl wp_admin_asset path_reg (^|/)wp-admin/(css|js|images)/(?!.*\.php).*$ + acl wp_gate_exempt hdr(host),lower -f /etc/haproxy/wpadmin_gate_exempt.list + # ENCODED SEPARATOR. percent-decode-unreserved deliberately does NOT decode + # %2F -- "/" is a reserved character, and decoding it in the normalizer + # would change the path's structure (it would invent new segments), which + # is precisely why HAProxy refuses to. But OpenLiteSpeed DOES decode it and + # then serves the file: /wp-admin%2Fplugins.php was measured returning 302 + # from a real WordPress site on the OLS tier, i.e. full PHP boot, while + # matching none of the ACLs above. Apache returns 404 for the same request + # (AllowEncodedSlashes Off), so this is an OLS-tier defect -- and OLS is the + # tier currently saturating. + # + # DENY, not "treat it as a wp-admin path and redirect". Two reasons: + # 1. The redirect target is computed by regsub(/wp-admin/.*) which finds + # no "/wp-admin/" in "/wp-admin%2Fplugins.php", so `path` would come + # back UNCHANGED and the Location would point at the request's own + # URL -- an infinite redirect loop, not a gate. + # 2. Nothing legitimate emits it. A path segment cannot contain a literal + # "/", so %2F inside a path is always either a probe or a proxy- + # confusion attempt, and the Apache tier has been 404ing it all along, + # so no site on the fleet can depend on it. + # + # SCOPED to paths that mention wp-admin, not all paths. A blanket "deny any + # %2F in any path" would also hit REST/API-style routes on non-WordPress + # customer apps that legitimately pass an encoded slash inside a path + # parameter. Scoping keeps the blast radius inside the attack surface this + # gate owns. + # + # Matching is on the SUBSTRING, not an anchored pattern, on purpose: + # /blog%2Fwp-admin/plugins.php hides the separator BEFORE "wp-admin", where + # an anchored (^|/)wp-admin/ never matches, and OLS still resolves it to + # /blog/wp-admin/plugins.php. Substring matching catches the separator + # wherever it is. percent-to-uppercase has already folded %2f into %2F; + # the -i is belt and braces so this rule stands on its own if the + # normalizer is ever reordered. + # + # %5C (encoded backslash) is denied on the same terms. On this fleet's + # Linux backends a backslash is an ordinary filename character, so + # /wp-admin%5Cplugins.php 404s rather than booting PHP -- measured, it is + # not a live bypass today. It is included because it is the same + # encoded-separator trick against a backend that happens to treat "\" as + # one, it costs nothing, and no legitimate path contains it. + acl wp_admin_word path -i -m sub wp-admin + acl path_has_encoded_sep path -i -m sub %2f %5c + http-request deny deny_status 403 if wp_admin_word path_has_encoded_sep !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted + http-request deny deny_status 403 if wp_admin_path !wp_admin_safe_path !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted + http-request set-var(txn.wp_login_url) path,regsub(/wp-admin/.*,/wp-login.php,i) if wp_admin_path + http-request redirect code 302 location %[var(txn.wp_login_url)]?redirect_to=%[path,url_enc] if wp_admin_path wp_admin_safe_path !wp_admin_allowed !wp_admin_asset !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted + # IP blocking using map file (manual blocks only) # Map file format: /etc/haproxy/blocked_ips.map contains " 1" per line # Runtime updates: echo "add map #0 IP_ADDRESS 1" | socat stdio /var/run/haproxy.sock diff --git a/wpadmin_gate_exempt.list b/wpadmin_gate_exempt.list new file mode 100644 index 0000000..ef7ae9e --- /dev/null +++ b/wpadmin_gate_exempt.list @@ -0,0 +1,16 @@ +# Per-site opt-out from the WordPress admin edge gate. +# +# Hostnames listed here are EXEMPT: unauthenticated /wp-admin/* requests for +# these sites pass through to PHP instead of being redirected to wp-login.php. +# One hostname per line, lowercase. Matched against the Host header. +# +# Referenced by templates/hap_listener.tpl: +# acl wp_gate_exempt hdr(host),lower -f /etc/haproxy/wpadmin_gate_exempt.list +# +# Add a site here when a plugin legitimately serves unauthenticated visitors +# from a /wp-admin/ URL that is not in the rule's allowlist. Symptom: "my +# plugin's admin page redirects to login". +# +# Do NOT commit real customer domains — this repo is mirrored publicly. Add +# entries directly on the server; the file lives in the /etc/haproxy named +# volume and persists across container recreates.