cloud-hosting-platform/haproxy-manager-base
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'ci/haproxy-config-gate'

Browse Source
This commit is contained in:
shadowdao
2026-08-14 09:55:57 -07:00
parent 992bf49138 17ca731ed3
commit 2148d72334
7 changed files with 1380 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+49 -1
View File
@@ -23,7 +23,23 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.licenses="MIT"
RUN apt update -y && apt dist-upgrade -y && apt install socat haproxy cron certbot curl jq net-tools -y && apt clean && rm -rf /var/lib/apt/lists/*
# haproxy is PINNED. It was previously unpinned, so the binary could move under
# us at Debian's timing — an upstream release that rejected our config would have
# broken an unrelated commit's build, or worse, shipped an edge that refuses to
# start (see the `haproxy -c` gate below for why that matters: a config HAProxy
# rejects leaves the container Up with 80/443 unbound and /health still 200).
#
# Pinning does NOT make the gate redundant, and the gate does NOT make pinning
# unnecessary — they compose. Pinned means the version moves deliberately; the
# gate then answers immediately whether the new binary still accepts fleet config.
# It also makes the image reproducible, which it previously was not.
#
# To move it: bump the version here, rebuild, and let the gate verify. If Debian
# security-updates the package (e.g. -1+deb13u4) the build FAILS until this pin is
# updated — that failure is the point, not a bug. Check availability with:
# apt-cache policy haproxy
ARG HAPROXY_VERSION=3.0.11-1+deb13u3
RUN apt update -y && apt dist-upgrade -y && apt install socat "haproxy=${HAPROXY_VERSION}" cron certbot curl jq net-tools -y && apt-mark hold haproxy && apt clean && rm -rf /var/lib/apt/lists/*
WORKDIR /haproxy
COPY ./templates /haproxy/templates
COPY requirements.txt /haproxy/
@@ -40,11 +56,43 @@ COPY trusted_ips.map /etc/haproxy/trusted_ips.map
# volume shadows /etc/haproxy with.
COPY cloudflare_ips.list /haproxy/defaults/cloudflare_ips.list
COPY trusted_proxies.list /haproxy/defaults/trusted_proxies.list
COPY wpadmin_gate_exempt.list /haproxy/defaults/wpadmin_gate_exempt.list
# Place errorfiles outside the volumed path; the HAProxy config references
# them by absolute path.
COPY errors /haproxy/errors
RUN chmod +x /haproxy/scripts/*
RUN pip install -r requirements.txt
# ---------------------------------------------------------------------------
# Build gate: no image ships unless the real haproxy binary accepts the config
# this image's templates actually produce.
#
# On 2026-08-14 a template change rendered fine, passed all 13 unit tests, and
# was rejected by HAProxy ("invalid arg 2 in converter 'regsub'"). It was only
# caught because someone built an image by hand and ran `haproxy -c`. Nothing
# in the build or in CI would have stopped it: .gitea/workflows/build-push.yaml
# is checkout -> build -> push, and test-config-rollback.py's "haproxy" is a
# shell stub that only rejects a sentinel token. In production an invalid
# haproxy.cfg means init.py refuses to start HAProxy while the container stays
# Up - ports 80/443 unbound, every site on the host down, /health still 200.
#
# This lives in the Dockerfile rather than in the workflow deliberately:
# * it cannot be skipped, and it protects local `docker build` too;
# * no workflow restructuring (build-push-action builds and pushes in one
# step, so gating in CI would mean splitting build from push);
# * it validates against the EXACT haproxy binary in this image. Line 26
# installs haproxy unpinned, so that binary moves between builds - this
# turns "the new haproxy rejects our config" from a silent production
# risk into a build failure.
#
# The unit suites run here too. They had never run anywhere automated either,
# and they cost a few seconds.
RUN python3 /haproxy/scripts/test-wpadmin-gate.py \
&& python3 /haproxy/scripts/test-trusted-proxy-gate.py \
&& python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \
&& python3 /haproxy/scripts/test-config-rollback.py \
&& python3 /haproxy/scripts/test-cert-write-safety.py \
&& python3 /haproxy/scripts/test-cert-scripts.py \
&& python3 /haproxy/scripts/validate-rendered-config.py
# Create log directories
RUN mkdir -p /var/log && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
RUN chmod 755 /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
scripts/start-up.sh
+12 -4
View File
@@ -27,11 +27,12 @@ mkdir -p /etc/haproxy
# existing hosts instead of being permanently shadowed by the volume.
# Overwrite it from the baked copy on every start.
#
# trusted_proxies.list is OPERATOR DATA: operators add entries directly on
# the server and those must survive restarts/recreates. Seed it from the
# baked copy only when it's missing; never overwrite an existing one.
# trusted_proxies.list and wpadmin_gate_exempt.list are OPERATOR DATA:
# operators add entries directly on the server and those must survive
# restarts/recreates. Seed each from the baked copy only when it's missing;
# never overwrite an existing one.
#
# Both branches fall back to an empty file if the baked default is somehow
# All branches fall back to an empty file if the baked default is somehow
# absent, because "acl ... -f <missing file>" is a fatal HAProxy config
# error -- the list files must exist unconditionally by the time HAProxy starts.
if [ -f /haproxy/defaults/cloudflare_ips.list ]; then
@@ -46,6 +47,13 @@ if [ ! -f /etc/haproxy/trusted_proxies.list ]; then
: > /etc/haproxy/trusted_proxies.list
fi
fi
if [ ! -f /etc/haproxy/wpadmin_gate_exempt.list ]; then
if [ -f /haproxy/defaults/wpadmin_gate_exempt.list ]; then
cp /haproxy/defaults/wpadmin_gate_exempt.list /etc/haproxy/wpadmin_gate_exempt.list
else
: > /etc/haproxy/wpadmin_gate_exempt.list
fi
fi
cron &
scripts/test-wpadmin-gate.py
+511
View File
@@ -0,0 +1,511 @@
#!/usr/bin/env python3
"""Regression tests for the WordPress admin edge gate in hap_listener.tpl.
Why this file exists
--------------------
Unauthenticated GETs to /wp-admin/* were reaching PHP, booting WordPress just to
produce a login redirect and exhausting lsphp pools under a distributed
low-and-slow attack. The gate redirects them at the edge instead.
Two properties are easy to get wrong and invisible to `haproxy -c`:
* ORDERING. `is_whitelisted` reads var(txn.real_ip). If the rule renders before
the set-var chain, the whitelist evaluates against an unset variable.
* THE ALLOWLIST. wp-login.php loads its OWN css/js from /wp-admin/. Dropping
those entries leaves every login page on the fleet unstyled, while still
returning 200 -- a silent regression.
A THIRD property, added after an adversarial mutation audit: every assertion
here must be scoped to the CODE, not the surrounding prose. This file's own
comment blocks quote ACL names, rule fragments and even whole rules to explain
them -- which means a bare `assertIn` / `re.search` / `str.index` run over the
raw rendered config passes just as happily when the real rule has been deleted
(or merely commented out) and only its explanation survives. The audit proved
this concretely: commenting out the entire redirect rule, or the
`acl wp_admin_allowed` line, or all five normalizers, left the previous version
of this file at 26/26 PASS. See `rule_lines()` below, and use it (or one of the
guarded helpers built on it) for every assertion about whether a rule exists,
what it says, or where it sits relative to another rule. Do not add a new
`self.cfg.index(...)`, `self.assertIn(x, self.cfg)`, or `re.search(pattern,
self.cfg)` to this file -- none of them can tell code from comment.
Running
-------
python3 scripts/test-wpadmin-gate.py
"""
import os
import re
import sys
import logging
import tempfile
import unittest
MODULE_DIR = os.path.abspath(
os.environ.get('HAPROXY_MANAGER_DIR',
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
)
os.chdir(MODULE_DIR)
sys.path.insert(0, MODULE_DIR)
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda filename, *a, **kw: _real_file_handler(
os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw)
)
import haproxy_manager # noqa: E402
ALLOWLIST = ('/admin-ajax.php', '/admin-post.php',
'/load-styles.php', '/load-scripts.php')
EXCLUSIONS = ('!wp_admin_allowed', '!wp_admin_asset', '!has_wp_logged_in',
'!wp_gate_exempt', '!is_local', '!is_trusted_ip', '!is_whitelisted')
# The normalizer set, in the order it MUST render. Decoding has to precede the
# path walkers or "%2e%2e" is decoded to ".." only after path-strip-dotdot has
# already run, leaving the ".." unresolved -- measured against real HAProxy
# 3.0.11, both orders side by side.
NORMALIZERS = ('percent-to-uppercase',
'percent-decode-unreserved',
'path-merge-slashes',
'path-strip-dot',
'path-strip-dotdot full')
def render_listener():
return haproxy_manager.template_env.get_template('hap_listener.tpl').render(
crt_path='/etc/haproxy/certs',
suspension_enabled=False,
coraza_spoe_backend=None,
)
def render_header():
return haproxy_manager.template_env.get_template('hap_header.tpl').render(
cluster_secret=None,
)
# ---------------------------------------------------------------------------
# Comment-safe config inspection
#
# Every helper below operates on `rule_positions()`'s output, never on the raw
# rendered string. That is the one rule this whole module exists to enforce on
# itself: a mutation audit found that commenting out a real rule (prefixing it
# with '#', or -- more slyly -- deleting it and appending its own text as a
# TRAILING comment on the line above) left the previous version of these tests
# fully green, because plain `str.index` / `assertIn` / `re.search` over
# `self.cfg` cannot distinguish code from a comment that merely quotes it.
# ---------------------------------------------------------------------------
def rule_positions(cfg, needle):
"""[(comment-stripped line, char offset in cfg)] for every non-comment
line containing `needle`, in document order.
Two things a bare substring/regex search over `self.cfg` gets wrong, both
fixed here:
1. A line that is ENTIRELY a comment (starts with '#' once stripped) is
dropped. This is necessary but not sufficient -- see (2).
2. A line that MIXES real config with a trailing comment
(`live-code # note`, or the decoy `live-code # was: <the other
rule's exact text>`) is truncated at the first ' #' before matching,
so text stuffed into a trailing comment cannot masquerade as the
rule itself. A real HAProxy comment always starts at a '#' preceded
by whitespace here -- none of these templates use bare '#' as a
value character -- so this truncation does not clip real rules.
Returning (line, position) pairs together -- rather than making callers
re-derive one from the other with a second `cfg.index(line)` -- also
avoids a subtler bug: if the same comment-stripped line occurs twice
(e.g. a duplicated rule), re-deriving the position with `str.index` always
finds the FIRST copy regardless of which one you meant. Walking the file
once and recording positions as we go keeps first/last unambiguous.
"""
out = []
pos = 0
for raw in cfg.split('\n'):
stripped = raw.strip()
if stripped and not stripped.startswith('#'):
code = stripped.split(' #', 1)[0].rstrip()
if code and needle in code:
out.append((code, pos))
pos += len(raw) + 1 # +1 for the '\n' split() consumed
return out
def rule_lines(cfg, needle):
"""Comment-stripped rule lines containing `needle` (text only, no
position). See `rule_positions()` for what this guards against. Every
assertion about a RULE's presence or content must go through this (or
`rule_positions`/the guarded helpers below) -- never a bare
`needle in cfg` or `re.search(pattern, cfg)`.
"""
return [line for line, _ in rule_positions(cfg, needle)]
def require_rule(cfg, needle, what=None):
"""The single rule line containing `needle`.
Raises a plain AssertionError naming what was being looked for -- not
IndexError from an unguarded `rule_lines(...)[0]`, and not
`ValueError: substring not found` from a bare `cfg.index(...)` -- when
the rule is missing. A missing rule and a broken test harness must not
look identical in a failure report.
Raises the same way if `needle` is ambiguous (matches more than one rule
line): silently taking the first match in that case would hide the
ambiguity instead of surfacing it.
"""
label = what or needle
lines = rule_lines(cfg, needle)
if not lines:
raise AssertionError('no rule found for %r (expected: %s)' % (needle, label))
if len(lines) > 1:
raise AssertionError(
'%r matched %d rule lines, expected exactly one (%s): %r'
% (needle, len(lines), label, lines))
return lines[0]
def require_position(cfg, needle, what=None, last=False):
"""(line, char offset) for an ordering assertion, guarded the same way as
`require_rule` -- but tolerant of the needle matching multiple lines
(e.g. a multi-line set-var "chain"), since ordering checks often want the
first or last of several. Pass last=True for the last occurrence.
"""
label = what or needle
positions = rule_positions(cfg, needle)
if not positions:
raise AssertionError(
'no rule found for %r, cannot check ordering (expected: %s)' % (needle, label))
return positions[-1] if last else positions[0]
class WpAdminGate(unittest.TestCase):
def setUp(self):
self.cfg = render_listener()
def test_wp_admin_path_acl_declared(self):
line = require_rule(self.cfg, 'acl wp_admin_path', 'wp_admin_path ACL')
self.assertIn('path_reg', line)
def test_wp_admin_asset_acl_declared(self):
line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL')
self.assertIn('path_reg', line)
def test_wp_admin_allowed_acl_declared(self):
line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL')
self.assertIn('path_end', line)
def test_wp_gate_exempt_acl_declared(self):
"""Scoped to the ACL line itself, not `self.cfg` as a whole -- the
surrounding prose (see hap_listener.tpl's "per-site opt-out" comment)
also spells out /etc/haproxy/wpadmin_gate_exempt.list verbatim, so an
unscoped `assertIn` would still pass with the real ACL deleted.
"""
line = require_rule(self.cfg, 'acl wp_gate_exempt', 'wp_gate_exempt ACL')
self.assertIn('/etc/haproxy/wpadmin_gate_exempt.list', line)
def test_allowlist_entries_present(self):
"""wp-login.php loads its own css/js from /wp-admin/ -- see module docstring."""
line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL')
for entry in ALLOWLIST:
with self.subTest(entry=entry):
self.assertIn(entry, line)
def test_static_asset_dirs_allowed(self):
line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL')
self.assertRegex(line, r'path_reg.*\(css\|js\|images\)')
def test_install_php_is_NOT_allowlisted(self):
"""install.php is deliberately gated -- a takeover vector on abandoned installs."""
line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL')
self.assertNotIn('install.php', line)
def test_redirect_rule_has_all_exclusions(self):
rule = require_rule_by_predicate(
self.cfg, 'http-request redirect', lambda ln: 'wp_admin_path' in ln,
'wp-admin redirect rule')
for excl in EXCLUSIONS:
with self.subTest(exclusion=excl):
self.assertIn(excl, rule)
def test_rule_renders_after_real_ip_resolution(self):
"""is_whitelisted reads txn.real_ip; before the set-var chain it is unset."""
_, setvar_pos = require_position(self.cfg, 'set-var(txn.real_ip)',
'real_ip set-var chain')
_, rule_pos = require_position(self.cfg, 'wp_admin_path', 'wp_admin_path ACL/rule')
self.assertLess(setvar_pos, rule_pos,
'wp_admin_path renders before txn.real_ip is resolved')
def test_rule_renders_after_has_wp_logged_in_declared(self):
"""HAProxy resolves ACLs as it parses; use-before-declare fails."""
_, decl_pos = require_position(self.cfg, 'acl has_wp_logged_in',
'has_wp_logged_in ACL declaration')
_, rule_pos = require_position(self.cfg, 'wp_admin_path', 'wp_admin_path ACL/rule')
self.assertLess(decl_pos, rule_pos,
'wp_admin_path renders before has_wp_logged_in is declared')
def test_only_one_has_wp_logged_in_declaration(self):
self.assertEqual(len(rule_lines(self.cfg, 'acl has_wp_logged_in')), 1)
def test_allowlist_entries_are_anchored_to_wp_admin(self):
"""Bare `path_end /admin-ajax.php` also matches
/wp-admin/evil/admin-ajax.php, which ALSO matches wp_admin_path
(path_reg only requires /wp-admin/ to appear somewhere) -- an
attacker-inserted path segment would then sail through the
allowlist ungated. Entries must be anchored to sit directly under
wp-admin/. Scoped to the captured ACL line only, since the
surrounding comment block also mentions these bare filenames.
"""
line = require_rule(self.cfg, 'acl wp_admin_allowed', 'wp_admin_allowed ACL')
for entry in ALLOWLIST:
with self.subTest(entry=entry):
self.assertIn('/wp-admin' + entry, line)
self.assertNotRegex(
line, r'(?<!wp-admin)' + re.escape(entry) + r'(?!\S)',
'found a bare, unanchored allowlist entry: ' + entry)
def test_wp_login_url_setvar_renders_in_correct_order(self):
"""The inline regsub-in-`location` form is rejected by real HAProxy
3.0.11 (invalid arg 2 in converter 'regsub'), so the regsub is
computed in its own set-var line instead. That set-var must render
after the set-var(txn.real_ip) chain (it must not disturb that
load-bearing chain) and before the redirect rule that consumes it.
"""
_, last_real_ip_pos = require_position(self.cfg, 'set-var(txn.real_ip)',
'real_ip set-var chain', last=True)
_, wp_login_pos = require_position(self.cfg, 'set-var(txn.wp_login_url)',
'wp_login_url set-var')
_, redirect_pos = require_position(
self.cfg, 'http-request redirect code 302 location %[var(txn.wp_login_url)]',
'wp-admin redirect rule')
self.assertLess(last_real_ip_pos, wp_login_pos,
'wp_login_url set-var must render after the real_ip set-var chain')
self.assertLess(wp_login_pos, redirect_pos,
'wp_login_url set-var must render before the redirect rule that uses it')
def test_redirect_rule_uses_setvar_not_inline_regsub(self):
"""Guards against reintroducing the rejected inline form."""
rule = require_rule_by_predicate(
self.cfg, 'http-request redirect', lambda ln: 'wp_admin_path' in ln,
'wp-admin redirect rule')
self.assertIn('%[var(txn.wp_login_url)]', rule)
self.assertNotIn('regsub', rule)
def test_redirect_rule_requires_safe_path(self):
"""OPEN REDIRECT guard. The redirect target is built by rewriting
`path` with regsub, which only replaces the matched substring --
everything before "/wp-admin/" survives untouched in the output.
Three concrete requests turn that into an off-site `Location:`
header: "//evil.example.com/wp-admin/x.php" (protocol-relative,
browsers resolve "//host/path" to "https://host/path"),
"/\\evil.example.com/wp-admin/x.php" (browsers normalise a leading
"/\\" the same as "//"), and an RFC 7230 absolute-form request
target ("https://evil.example.com/wp-admin/x.php") which can make
HAProxy's `path` fetch return a full URI. wp_admin_safe_path
(requiring a well-formed absolute path) must be a POSITIVE
condition on the redirect rule -- scoped to the captured rule line
only, since the surrounding comment block also mentions this ACL
name and a bare substring match would pass even if the condition
were dropped from the rule itself.
"""
rule = require_rule_by_predicate(
self.cfg, 'http-request redirect', lambda ln: 'wp_admin_path' in ln,
'wp-admin redirect rule')
self.assertIn('wp_admin_safe_path', rule)
self.assertNotIn('!wp_admin_safe_path', rule,
'wp_admin_safe_path must be a positive condition, not negated')
def test_wp_admin_safe_path_acl_declared(self):
line = require_rule(self.cfg, 'acl wp_admin_safe_path', 'wp_admin_safe_path ACL')
self.assertIn('path_reg', line)
def test_unsafe_wp_admin_path_is_denied_not_passed_through(self):
"""wp_admin_safe_path being a POSITIVE condition on the redirect means
a path that fails it is simply not redirected -- which used to mean it
fell through to the backend UNGATED, i.e. exactly the PHP-booting
request the gate exists to stop. Normalisation removes the "//"
spelling of that, but not "/\\", so the fall-through must be closed
with an explicit deny rather than left implicit.
"""
denies = rule_lines(self.cfg, '!wp_admin_safe_path')
self.assertTrue(denies,
'no rule denies a wp-admin path that fails wp_admin_safe_path')
self.assertTrue(any(d.startswith('http-request deny') for d in denies),
'the !wp_admin_safe_path rule must be a deny: %r' % denies)
def require_rule_by_predicate(cfg, needle, predicate, what):
"""Like require_rule(), but for rules identified by needle + a predicate
over the comment-stripped line (e.g. "the `http-request redirect` line
that also mentions wp_admin_path", since the frontend has more than one
`http-request redirect`). Raises a clear AssertionError, not IndexError
or a silently-empty match, if no line satisfies both.
"""
candidates = [ln for ln in rule_lines(cfg, needle) if predicate(ln)]
if not candidates:
raise AssertionError('no rule found matching %s' % what)
if len(candidates) > 1:
raise AssertionError('%s matched more than one rule line: %r' % (what, candidates))
return candidates[0]
class UriNormalisation(unittest.TestCase):
"""The gate matches the RAW path; the backend normalises and decodes it.
Every gap between those is a bypass -- five were found this way. These
tests pin the normalisation that closes the gap as a class.
NOTE: these are config-TEXT assertions. They are necessary but NOT
sufficient: the previous revision of this file passed while five live
bypasses shipped. The real evidence is the behavioural matrix run against
real haproxy 3.0.11 with raw sockets -- see
.superpowers/sdd/2026-08-14-wpadmin-edge-gate/task-4-normalize-report.md.
"""
def setUp(self):
self.cfg = render_listener()
self.header = render_header()
def test_experimental_directives_exposed_in_global(self):
"""normalize-uri is experimental in 3.0; without this HAProxy refuses
to start (`haproxy -c` exits 1, ALERT). That does NOT crash-loop the
container, though -- see hap_header.tpl's comment and
haproxy_manager.py's start_haproxy()/do_initial_setup(): the failure
is swallowed, and the container comes up with haproxy simply never
running. This test exists so that silent-outage mode is never
reintroduced by dropping this line.
"""
lines = rule_lines(self.header, 'expose-experimental-directives')
matches = [ln for ln in lines if ln == 'expose-experimental-directives']
self.assertTrue(
matches, 'expose-experimental-directives missing from the global section')
def test_all_normalizers_render(self):
for norm in NORMALIZERS:
with self.subTest(normalizer=norm):
self.assertTrue(
rule_lines(self.cfg, 'normalize-uri ' + norm),
'missing normalizer: ' + norm)
def test_normalizer_order_decode_before_path_walkers(self):
"""Reverse this order and /wp-admin/js/%2e%2e/plugins.php reaches the
ACLs as /wp-admin/js/../plugins.php -- decoded but unresolved.
"""
lines = [ln for ln in rule_lines(self.cfg, 'http-request normalize-uri')
if ln.startswith('http-request normalize-uri')]
names = [ln.split('normalize-uri ', 1)[1] for ln in lines]
self.assertEqual(
names, list(NORMALIZERS),
'normalize-uri directives are missing, reordered, or duplicated: %r' % names)
def test_normalisation_precedes_every_path_based_rule(self):
"""A normalizer placed after a path rule normalises nothing for it."""
norm_positions = []
for n in NORMALIZERS:
_, pos = require_position(self.cfg, 'http-request normalize-uri ' + n,
'normalizer: ' + n)
norm_positions.append(pos)
last_norm = max(norm_positions)
for marker in ('acl is_health_check', 'acl wp_login_path',
'acl xmlrpc_path', 'acl wp_batch_path',
'acl wp_admin_path', 'http-request set-path'):
with self.subTest(rule=marker):
_, marker_pos = require_position(self.cfg, marker, marker)
self.assertLess(last_norm, marker_pos,
marker + ' renders before URI normalisation')
def test_query_sort_by_name_is_not_enabled(self):
"""query-sort-by-name reorders query parameters, which would break
anything that signs or caches on the exact query string. This is NOT
because the enabled normalizers already leave the query alone --
percent-to-uppercase and percent-decode-unreserved rewrite the WHOLE
request-target, query string included (see hap_listener.tpl's BLAST
RADIUS comment) -- it is a deliberate line between "case-fold /
decode" (no-ops under RFC 3986) and "reorder" (not a no-op for a
signed/cached query string).
"""
self.assertFalse(rule_lines(self.cfg, 'normalize-uri query-sort-by-name'))
def test_dotdot_normalizer_uses_full(self):
"""Without "full", ".." segments that climb above the root are left in
place and /../../wp-admin/plugins.php survives -- measured.
"""
lines = rule_lines(self.cfg, 'normalize-uri path-strip-dotdot')
self.assertTrue(lines, 'missing normalizer: path-strip-dotdot')
for ln in lines:
self.assertTrue(ln.endswith('path-strip-dotdot full'), ln)
def test_encoded_separator_on_wp_admin_is_denied(self):
"""percent-decode-unreserved deliberately leaves %2F encoded ("/" is
reserved), but OpenLiteSpeed decodes it and serves the file --
/wp-admin%2Fplugins.php was measured booting PHP on the OLS tier while
matching no wp-admin ACL. Normalisation cannot close this; it needs its
own rule.
"""
denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep')
if ln.startswith('http-request deny')]
self.assertTrue(denies, 'no deny rule for encoded separators')
acl_line = require_rule(self.cfg, 'acl path_has_encoded_sep', 'path_has_encoded_sep ACL')
self.assertIn('%2f', acl_line.lower())
def test_encoded_separator_deny_is_scoped_to_wp_admin(self):
"""A blanket "deny any %2F in any path" would break non-WordPress
customer apps that legitimately pass an encoded slash in a path
parameter. The deny must be conditioned on the path mentioning
wp-admin.
"""
denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep')
if ln.startswith('http-request deny')]
self.assertTrue(denies, 'no deny rule for encoded separators')
for d in denies:
self.assertIn('wp_admin_word', d)
def test_encoded_separator_deny_honors_the_same_whitelist(self):
denies = [ln for ln in rule_lines(self.cfg, 'path_has_encoded_sep')
if ln.startswith('http-request deny')]
self.assertTrue(denies, 'no deny rule for encoded separators')
for d in denies:
for excl in ('!has_wp_logged_in', '!wp_gate_exempt', '!is_local',
'!is_trusted_ip', '!is_whitelisted'):
with self.subTest(rule=d, exclusion=excl):
self.assertIn(excl, d)
def test_encoded_separator_acl_matches_a_substring_not_a_prefix(self):
"""/blog%2Fwp-admin/plugins.php hides the separator BEFORE "wp-admin",
where an anchored pattern never matches, and OLS still resolves it.
"""
acl_line = require_rule(self.cfg, 'acl path_has_encoded_sep', 'path_has_encoded_sep ACL')
self.assertIn('-m sub', acl_line)
def test_wp_admin_asset_bypass_cannot_cover_a_php_entrypoint(self):
"""The asset bypass anchored its prefix but not its suffix, so
/wp-admin/css/../plugins.php took it and the backend then resolved
".." and booted plugins.php. path-strip-dotdot is the real fix; this
keeps the bypass structurally incapable of covering PHP.
"""
acl_line = require_rule(self.cfg, 'acl wp_admin_asset', 'wp_admin_asset ACL')
self.assertIn('.php', acl_line,
'wp_admin_asset must exclude .php explicitly')
def test_case_insensitive_acl_and_regsub_are_kept_in_sync(self):
"""A case-insensitive wp_admin_path with a case-sensitive regsub is an
INFINITE REDIRECT LOOP: regsub finds no "/wp-admin/" in
"/WP-ADMIN/plugins.php", returns `path` unchanged, and the Location
then points at the request's own URL.
"""
acl_line = require_rule(self.cfg, 'acl wp_admin_path', 'wp_admin_path ACL')
setvar_line = require_rule(self.cfg, 'set-var(txn.wp_login_url)', 'wp_login_url set-var')
acl_ci = bool(re.search(r'path_reg\s+-i\s', acl_line))
regsub_ci = bool(re.search(r'regsub\([^)]*,\s*i\)', setvar_line))
self.assertEqual(
acl_ci, regsub_ci,
'wp_admin_path case-sensitivity (%s) and regsub flags (%s) disagree'
% (acl_line, setvar_line))
if __name__ == '__main__':
unittest.main(verbosity=2)
scripts/validate-rendered-config.py
+423
View File
@@ -0,0 +1,423 @@
#!/usr/bin/env python3
"""Build gate: render the real HAProxy config and hand it to the real `haproxy -c`.
Why this file exists
--------------------
On 2026-08-14 a change to hap_listener.tpl rendered perfectly, passed every
unit test in scripts/ (13 green), and was then rejected outright by HAProxy:
[ALERT] config : parsing [/etc/haproxy/haproxy.cfg:...] :
invalid arg 2 in converter 'regsub' : ... unexpected empty
replacement string
Nothing between "commit" and "running in production" would have caught it.
The unit tests assert on the *text* of the rendered config with regexes, which
tells you what the template says, never whether HAProxy will accept it. And
scripts/test-config-rollback.py stubs the `haproxy` binary with a shell script
that only rejects a literal sentinel token, so its "validation" has never
parsed a single line of real HAProxy syntax.
The failure mode this guards is not cosmetic. When haproxy.cfg is invalid,
scripts/init.py refuses to start HAProxy but the container still comes up:
ports 80/443 are unbound, every site on the host is down, and /health keeps
answering 200 because the Flask API is fine.
So: render the config through the SAME code path production uses
(haproxy_manager.generate_config(), templates and all), then run the actual
`haproxy -c` against the result and gate on its exit code.
This runs as a RUN step in the Dockerfile, which means it also validates
against the exact haproxy binary that ships in the image being built - note
that the Dockerfile installs haproxy UNPINNED, so that binary can move under
us between builds. Any syntax the new binary rejects now fails the build
instead of failing at 3am on an edge node.
What it covers
--------------
* every template generate_config() touches, assembled in the real order
* a domain with SSL + a backend, a wildcard domain, a cert-only domain with
no backend, and two template_override backends
* blocked-IP map entries (single IP and CIDR)
* BOTH sides of the two conditional blocks in hap_listener.tpl -
{%- if suspension_enabled %} and {%- if coraza_spoe_backend %} - because a
syntax error inside a conditional ships undetected otherwise. Scenario
"full" turns both on; scenario "default" leaves both off, which is the
byte-identical-to-standalone shape.
Warnings vs failures
--------------------
`haproxy -c` emits warnings on a clean config here (at minimum "Can't load
stats file" because /var/lib/haproxy/stats.dat doesn't exist at build time,
plus assorted path_reg/ACL advisories). Those are NOT failures. This gate keys
on the process EXIT CODE only, and dumps the full output when it is non-zero.
Running
-------
python3 scripts/validate-rendered-config.py
Needs the real `haproxy` binary, the application's Python dependencies, and
write access to /etc/haproxy (several templates reference files there by
absolute path - see _REAL_PATH_NOTE below). Inside the image build all three
hold. On a workstation, run it in the container instead.
"""
import logging
import os
import re
import shutil
import sqlite3
import subprocess
import sys
import tempfile
MODULE_DIR = os.path.abspath(
os.environ.get('HAPROXY_MANAGER_DIR',
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
)
os.chdir(MODULE_DIR)
sys.path.insert(0, MODULE_DIR)
# Same trick the other suites use: haproxy_manager configures logging at import
# time against /var/log/haproxy-manager.log. Redirect the handlers so this runs
# without root and without polluting the image's log files.
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-validate-logs-')
_real_file_handler = logging.FileHandler
logging.FileHandler = (
lambda filename, *a, **kw: _real_file_handler(
os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw)
)
try:
import haproxy_manager as hm # noqa: E402
finally:
logging.FileHandler = _real_file_handler
# The application logs a lot at INFO during a render, and legitimately logs at
# ERROR about things that are only true in this harness ("no existing HAProxy
# config on disk ... ROLLBACK IS NOT AVAILABLE" - correct, there is no live
# config during a build). Silence it so the build log carries the gate's own
# verdict and haproxy's output, which is what matters.
logging.getLogger('haproxy_manager').setLevel(logging.CRITICAL)
# _REAL_PATH_NOTE
# ---------------
# Most paths haproxy_manager writes to are module-level constants and are
# redirected into a temp dir below. Two cannot be:
#
# /etc/haproxy/blocked_ips.map - hardcoded inside hap_listener.tpl's
# map_ip() converter
# /etc/haproxy/coraza-spoe.cfg - hardcoded in the `filter spoe engine`
# line, and parsed by haproxy -c
#
# Redirecting the constants without editing the templates would just make
# haproxy read a different (missing) file, so those two are left at their real
# paths. Everything this script creates under /etc/haproxy is removed again on
# exit; pre-existing files (the baked trusted_ips.*) are never touched.
ETC_HAPROXY = '/etc/haproxy'
# Files referenced with `-f` / map_ip() from the templates. A missing `-f` file
# is a FATAL haproxy error, so a gate that didn't create these would fail for
# reasons that have nothing to do with the config being tested.
STUB_FILES = {
os.path.join(ETC_HAPROXY, 'trusted_ips.list'): '# validation stub\n203.0.113.10\n',
os.path.join(ETC_HAPROXY, 'trusted_ips.map'): '# validation stub\n203.0.113.11 1\n',
os.path.join(ETC_HAPROXY, 'cloudflare_ips.list'): '# validation stub\n198.51.100.0/24\n',
os.path.join(ETC_HAPROXY, 'trusted_proxies.list'): '# validation stub\n192.0.2.0/24\n',
os.path.join(ETC_HAPROXY, 'wpadmin_gate_exempt.list'): '# validation stub\nexempt.example.test\n',
os.path.join(ETC_HAPROXY, 'suspended_domains.list'): 'suspended.example.test\n',
# `lf-file` on the Coraza deny rule; loaded at parse time. Present in the
# image (COPY errors /haproxy/errors), stubbed for anything else.
'/haproxy/errors/403-waf.html': '<html><body>blocked %[unique-id]</body></html>\n',
}
# (suspension_enabled, coraza_spoe_backend) combinations to render + validate.
SCENARIOS = (
('default', {}),
('full', {
'HAPROXY_SUSPENSION_ENABLED': 'true',
'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000',
}),
)
def log(msg):
sys.stdout.write(f'[validate-config] {msg}\n')
sys.stdout.flush()
def fail(msg):
sys.stderr.write(f'[validate-config] FAIL: {msg}\n')
sys.stderr.flush()
raise SystemExit(1)
class CreatedFiles:
"""Tracks what we put on disk outside the temp dir so it can be removed.
Two sources: files we create explicitly, and files generate_config() itself
writes into /etc/haproxy (blocked_ips.map, coraza-spoe.cfg and their
.backup copies). The latter are caught by diffing the directory listing,
which also picks up anything a future change starts writing there.
"""
def __init__(self):
self.explicit = []
self.etc_before = self._listdir(ETC_HAPROXY)
@staticmethod
def _listdir(path):
try:
return set(os.listdir(path))
except OSError:
return set()
def ensure(self, path, content):
"""Create path with content if it does not already exist."""
if os.path.exists(path):
return
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as fh:
fh.write(content)
os.chmod(path, 0o644)
self.explicit.append(path)
def cleanup(self):
for path in self.explicit:
try:
os.unlink(path)
except OSError:
pass
for name in self._listdir(ETC_HAPROXY) - self.etc_before:
try:
os.unlink(os.path.join(ETC_HAPROXY, name))
except OSError:
pass
def require_haproxy_binary():
"""Fail closed. A gate that skips itself when the binary is missing is a
gate that would have let the 2026-08-14 change through."""
path = shutil.which('haproxy')
if not path:
fail('no `haproxy` binary on PATH - this gate cannot validate anything. '
'Run it inside the image (the Dockerfile installs haproxy).')
version = subprocess.run([path, '-v'], capture_output=True, text=True)
log(f'using {path}: {version.stdout.strip().splitlines()[0] if version.stdout else "unknown version"}')
def make_self_signed_cert(certs_dir):
"""HAProxy loads every file in the `bind ... ssl crt <dir>` directory at
parse time, so the directory has to hold a real, loadable bundle."""
os.makedirs(certs_dir, exist_ok=True)
cert = os.path.join(certs_dir, 'cert.tmp')
key = os.path.join(certs_dir, 'key.tmp')
subprocess.run(
['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes',
'-keyout', key, '-out', cert, '-days', '1',
'-subj', '/CN=validate.example.test'],
check=True, capture_output=True,
)
bundle = os.path.join(certs_dir, 'validate.example.test.pem')
with open(bundle, 'w') as out:
for part in (cert, key):
with open(part) as fh:
out.write(fh.read())
os.unlink(cert)
os.unlink(key)
os.chmod(bundle, 0o600)
def seed_database(db_path):
"""A representative fleet: SSL + backend, wildcard, cert-only, overrides."""
hm.DB_FILE = db_path
hm.init_db()
with sqlite3.connect(db_path) as conn:
cur = conn.cursor()
def add_site(domain, backend, ssl_enabled=1, wildcard=0, override=None,
servers=(('web1', '10.0.0.10', 8080, 'check'),)):
cur.execute(
'INSERT INTO domains (domain, ssl_enabled, ssl_cert_path, '
'template_override, is_wildcard) VALUES (?, ?, ?, ?, ?)',
(domain, ssl_enabled, f'/etc/haproxy/certs/{domain}.pem',
override, wildcard),
)
domain_id = cur.lastrowid
cur.execute('INSERT INTO backends (name, domain_id, settings) '
'VALUES (?, ?, ?)', (backend, domain_id, None))
backend_id = cur.lastrowid
for name, addr, port, opts in servers:
cur.execute(
'INSERT INTO backend_servers (backend_id, server_name, '
'server_address, server_port, server_options) '
'VALUES (?, ?, ?, ?, ?)',
(backend_id, name, addr, port, opts),
)
add_site('site-one.example.test', 'site-one',
servers=(('web1', '10.0.0.10', 8080, 'check'),
('web2', '10.0.0.11', 8080, 'check backup')))
add_site('site-two.example.test', 'site-two', ssl_enabled=0)
add_site('*.wildcard.example.test', 'wildcard-site', wildcard=1)
add_site('ws.example.test', 'ws-site', override='hap_backend_websocket')
add_site('sse.example.test', 'sse-site', override='hap_backend_longlived')
# Cert/management-only domain: registered for certificates, no backend.
# generate_config() has an explicit branch for this.
cur.execute(
'INSERT INTO domains (domain, ssl_enabled, ssl_cert_path, '
'template_override, is_wildcard) VALUES (?, ?, ?, ?, ?)',
('panel.example.test', 1, '/etc/haproxy/certs/panel.pem', None, 0),
)
# Both map_ip() shapes: a single address and a CIDR.
for ip in ('203.0.113.66', '198.51.100.0/24'):
cur.execute('INSERT INTO blocked_ips (ip_address, reason, blocked_by) '
'VALUES (?, ?, ?)', (ip, 'validation fixture', 'gate'))
conn.commit()
def render(scenario_name, env_overrides, workdir):
"""Render via generate_config() and return the path to the assembled config.
generate_config() is the real production entry point: it takes the rollback
snapshot, writes the blocked-IP map, renders every template, and writes
haproxy.cfg. Only the reload is stubbed - there is no HAProxy process to
reload during a build, and validating the file is the whole point.
"""
scenario_dir = os.path.join(workdir, scenario_name)
certs_dir = os.path.join(scenario_dir, 'certs')
os.makedirs(scenario_dir)
make_self_signed_cert(certs_dir)
hm.HAPROXY_CONFIG_PATH = os.path.join(scenario_dir, 'haproxy.cfg')
hm.HAPROXY_BACKUP_PATH = os.path.join(scenario_dir, 'haproxy.cfg.backup')
hm.CLUSTER_SECRET_PATH = os.path.join(scenario_dir, 'cluster-secret')
hm.HAPROXY_SOCKET_PATH = os.path.join(scenario_dir, 'haproxy.sock')
hm.SSL_CERTS_DIR = certs_dir
seed_database(os.path.join(scenario_dir, 'haproxy_config.db'))
saved_env = {}
for key in ('HAPROXY_SUSPENSION_ENABLED', 'HAPROXY_CORAZA_SPOE_BACKEND'):
saved_env[key] = os.environ.pop(key, None)
os.environ.update(env_overrides)
real_reload = hm.reload_haproxy_safely
hm.reload_haproxy_safely = lambda *a, **kw: (True, 'reload skipped: build-time validation')
try:
hm.generate_config()
finally:
hm.reload_haproxy_safely = real_reload
for key, value in saved_env.items():
os.environ.pop(key, None)
if value is not None:
os.environ[key] = value
return hm.HAPROXY_CONFIG_PATH
def assert_scenario_branches(scenario_name, config_path, env_overrides):
"""Cheap sanity check that the conditional blocks actually rendered.
Without this, a template refactor that silently stopped emitting the
suspension or Coraza block would leave the gate passing while covering
less than it claims to.
"""
with open(config_path) as fh:
text = fh.read()
expected = {
'suspension': ('acl is_suspended_domain',
'HAPROXY_SUSPENSION_ENABLED' in env_overrides),
'coraza': ('filter spoe engine coraza',
'HAPROXY_CORAZA_SPOE_BACKEND' in env_overrides),
}
for label, (needle, should_be_present) in expected.items():
present = needle in text
if present != should_be_present:
fail(f'[{scenario_name}] {label} block {"missing" if should_be_present else "unexpectedly present"} '
f'in the rendered config (looked for {needle!r}). The gate is '
f'not covering what it thinks it is.')
def _dump_context(config_path, output):
"""Print the rendered lines HAProxy complained about.
The temp dir is deleted on the way out, so the build log has to carry the
evidence. HAProxy reports `parsing [<file>:<line>]`; show a window around
each reported line rather than dumping ~1500 lines of config.
"""
line_numbers = sorted({
int(n) for n in re.findall(
r'parsing \[' + re.escape(config_path) + r':(\d+)\]', output)
})
if not line_numbers:
return
with open(config_path) as fh:
lines = fh.read().splitlines()
sys.stderr.write('[validate-config] --- rendered config around the error ---\n')
for number in line_numbers:
start = max(1, number - 6)
end = min(len(lines), number + 4)
for index in range(start, end + 1):
marker = '>>' if index == number else ' '
sys.stderr.write(f'{marker}{index:6d}| {lines[index - 1]}\n')
sys.stderr.write('[validate-config] ---\n')
def validate(scenario_name, config_path):
result = subprocess.run(['haproxy', '-c', '-f', config_path],
capture_output=True, text=True)
if result.returncode != 0:
output = result.stdout + result.stderr
sys.stderr.write(
f'\n[validate-config] ===== {scenario_name}: haproxy REJECTED the '
f'rendered configuration (exit {result.returncode}) =====\n')
sys.stderr.write(output if output.endswith('\n') else output + '\n')
_dump_context(config_path, output)
sys.stderr.write(
'[validate-config] This is a real HAProxy parse failure. Shipping it '
'would leave the container Up with ports 80/443 unbound and every '
'site on the host down, while /health still returns 200.\n')
sys.stderr.flush()
raise SystemExit(1)
# Exit code 0 is the verdict. Warnings are expected and are NOT failures:
# "Can't load stats file" always fires at build time, and HAProxy emits
# path_reg/ACL advisories on a perfectly valid config.
noise = (result.stdout + result.stderr).strip()
log(f'{scenario_name}: haproxy -c OK (exit 0)')
if noise:
for line in noise.splitlines():
log(f' {scenario_name}: haproxy said: {line}')
def main():
require_haproxy_binary()
if not os.path.isdir(ETC_HAPROXY) or not os.access(ETC_HAPROXY, os.W_OK):
fail(f'{ETC_HAPROXY} must exist and be writable - several templates '
'reference files there by absolute path. Run this inside the image.')
created = CreatedFiles()
workdir = tempfile.mkdtemp(prefix='haproxy-validate-')
try:
for path, content in STUB_FILES.items():
created.ensure(path, content)
for scenario_name, env_overrides in SCENARIOS:
log(f'rendering scenario "{scenario_name}" '
f'({env_overrides or "no optional features"})')
config_path = render(scenario_name, env_overrides, workdir)
assert_scenario_branches(scenario_name, config_path, env_overrides)
validate(scenario_name, config_path)
finally:
created.cleanup()
shutil.rmtree(workdir, ignore_errors=True)
shutil.rmtree(_LOG_DIR, ignore_errors=True)
log('all scenarios accepted by the real haproxy binary')
return 0
if __name__ == '__main__':
raise SystemExit(main())
templates/hap_header.tpl
+30
View File
@@ -27,6 +27,36 @@ global
# SSL and Performance
tune.ssl.default-dh-param 2048
# Required by the `http-request normalize-uri` chain at the top of the
# `web` frontend (hap_listener.tpl). normalize-uri is still flagged
# EXPERIMENTAL in HAProxy 3.0, and HAProxy REFUSES TO START without this
# opt-in -- not a warning, a fatal:
# [ALERT] config : parsing [...] : 'normalize-uri' action is
# experimental, must be allowed via a global
# 'expose-experimental-directives'
# (verified against real haproxy 3.0.11-9e587df: `haproxy -c` exits 1).
# So this line and the normalize-uri rules must be added/removed together.
#
# Dropping this one alone does NOT crash-loop the container -- the truth
# is worse: it is a SILENT TOTAL OUTAGE that nothing escalates. Container
# init (scripts/init.py -> haproxy_manager.do_initial_setup()) calls
# generate_config() (which still succeeds -- Jinja doesn't validate
# HAProxy semantics) and then start_haproxy(), which runs `haproxy -c`,
# sees it fail, logs an error, and RETURNS WITHOUT RAISING. init.py exits
# 0. scripts/start-up.sh then execs gunicorn as PID 1 regardless. Result:
# the container stays "Up", ports 80/443 are never bound, EVERY SITE ON
# THE HOST IS DOWN, and the in-container supervisor loop
# (ensure_haproxy.py, every HAPROXY_SUPERVISOR_INTERVAL seconds) retries
# the identical failing render forever without ever escalating. Worse
# still, GET /health keeps returning HTTP 200 -- health_check() only
# answers 500 on a database error; a dead haproxy just flips the JSON
# body's "haproxy_status" to "stopped" while the status code a naive
# monitor checks never changes. Do not trust /health alone to catch this.
#
# This exposes ONLY the experimental directives that are actually used --
# it does not change the behaviour of anything else in this file.
expose-experimental-directives
# HTTP/3 over QUIC. The Debian haproxy package is built against system
# OpenSSL via the compatibility shim (USE_QUIC_OPENSSL_COMPAT), which is
# not a native QUIC TLS stack. HAProxy therefore rejects `quic*@` binds
templates/hap_listener.tpl
+339
View File
@@ -22,6 +22,103 @@ frontend web
# Capture Host header so it appears in httplog output (in %hr field)
http-request capture req.hdr(Host) len 64
# --- URI normalisation (MUST be the first path-touching block here) ---
# Every path-based control in this frontend (the ACME health-check bypass,
# wp-login, xmlrpc, the wp-json/batch virtual patch, the wp-admin gate, the
# blocked-IP and suspension set-path rules, and everything Coraza inspects)
# matched the RAW request-target while the backend NORMALISED and DECODED
# it before resolving a file. Every gap between those two behaviours is a
# bypass, and each one had to be patched individually. Five were found in
# the wp-admin gate alone, all the same class:
#
# //wp-admin/plugins.php raw path starts "//" -> the safe-path
# guard failed -> request fell through
# ungated, backend served it
# /wp-admin/css/../plugins.php matched the css/js/images asset
# bypass; backend resolved ".." and
# booted plugins.php
# /wp-admin/js/%2e%2e/plugins.php same, with the ".." percent-encoded
# /wp%2Dadmin/plugins.php "wp-admin" spelled with %2D never
# matched any wp-admin ACL at all
# /wp-admin%2Fplugins.php encoded separator; see the dedicated
# rule in the wp-admin gate below
#
# Rather than keep bolting a counter-pattern onto each rule, normalise the
# URI once, here, so every rule below matches the SAME string the backend
# will resolve. HAProxy rewrites the request-target in place, so the
# backend receives the normalised form too.
#
# ORDER IS LOAD-BEARING and was determined empirically against real
# haproxy 3.0.11, not from the docs. The decoders must run BEFORE the path
# walkers: with the reverse order, /wp-admin/js/%2e%2e/plugins.php ends up
# as /wp-admin/js/../plugins.php -- decoded, but the ".." left unresolved,
# because path-strip-dotdot had already run by the time the "%2e%2e"
# became "..". Verified both directions side by side.
#
# percent-to-uppercase %2f -> %2F. Canonicalises the spelling of
# whatever stays encoded, so downstream rules
# need one case of each escape, not two.
# percent-decode-unreserved Decodes ONLY RFC 3986 unreserved chars
# (A-Za-z0-9-._~). This is what turns %2e%2e
# into .. and wp%2Dadmin into wp-admin.
# Reserved escapes are deliberately left
# alone -- %2F in particular, which is why
# the wp-admin gate needs its own encoded-
# slash rule (see below).
# path-merge-slashes //x -> /x. Also removes the entire class of
# "leading // defeats an anchored regex".
# path-strip-dot /a/./b -> /a/b.
# path-strip-dotdot full /a/b/../c -> /a/c. "full" additionally
# resolves ".." segments that would climb
# above the root (/../../wp-admin/x.php ->
# /wp-admin/x.php); without "full" HAProxy
# leaves those in place and the vector
# survives -- measured, both forms tested.
#
# DELIBERATELY NOT ENABLED: query-sort-by-name. Reordering query-string
# parameters would silently break anything that signs or caches on the
# exact query string (signed asset URLs, HMAC'd callbacks, CDN cache
# keys). This is NOT because the enabled normalizers already leave the
# query alone -- see BLAST RADIUS just below, they don't -- it is a
# deliberate line drawn between "case-fold / decode", which RFC 3986
# defines as no-ops, and "reorder", which is not a no-op for a caller
# treating the query as an opaque signed string.
#
# BLAST RADIUS: this block applies to EVERY request for EVERY site on
# EVERY tier, so the decoding was kept minimal on purpose -- and it is
# NOT scoped to the path. percent-to-uppercase and percent-decode-
# unreserved normalise the WHOLE request-target as HAProxy parses it --
# query string included, not just the path component the ACLs below
# match on -- and the BACKEND receives the rewritten query on the wire,
# not just an internal haproxy view of it. Measured against real HAProxy
# 3.0.11:
# /a?sig=%2babc%2fdef -> /a?sig=%2Babc%2Fdef (percent-hex upper-cased)
# /a?b=%41%42%43 -> /a?b=ABC (unreserved chars decoded)
# /a?tok=%7e%2d%5f%2e -> /a?tok=~-_. (unreserved chars decoded)
# Only parameter ORDER is preserved -- that guarantee is exactly why
# query-sort-by-name above is the one normalizer in this family left
# disabled. Per RFC 3986 both enabled rewrites are defined as the same
# URI (case in a percent-escape, and an unreserved character vs. its
# escape, carry no distinct meaning), but "the same URI" is not "the
# same bytes": an application that HMACs or otherwise signs the RAW
# query string, rather than parsing it first, could see a mutated value
# and fail to verify an otherwise-legitimate request. Checked against
# this fleet: no .NET backends (.NET's UrlEncode emits lowercase
# percent-hex, which percent-to-uppercase would rewrite) and no
# URL-in-path proxies, so there is no known victim today -- but do not
# assume "path only" from this block; that was the actual bug in an
# earlier draft of this comment.
#
# normalize-uri is EXPERIMENTAL in 3.0 and requires
# `expose-experimental-directives` in the global section
# (hap_header.tpl). Without it HAProxy does not start. Remove one and you
# must remove the other.
http-request normalize-uri percent-to-uppercase
http-request normalize-uri percent-decode-unreserved
http-request normalize-uri path-merge-slashes
http-request normalize-uri path-strip-dot
http-request normalize-uri path-strip-dotdot full
# --- Trusted-proxy gate (MUST precede real-IP resolution below) ---
# CF-Connecting-IP / X-Real-IP / X-Forwarded-For are client-supplied. Any
# peer that is not a known reverse proxy gets them stripped, so the
@@ -183,6 +280,248 @@ frontend web
http-request deny deny_status 403 if wp_batch_route !has_wp_logged_in !is_local !is_trusted_ip !is_whitelisted
http-request deny deny_status 403 if wp_batch_route_enc !has_wp_logged_in !is_local !is_trusted_ip !is_whitelisted
# --- WordPress admin edge gate ---
# Measured on whp02, 2026-08-14: distributed unauthenticated GETs booting
# WordPress just to bounce back a login redirect --
# 2801 GET /wp-admin/profile.php 2781 GET /wp-admin/edit.php 2761 GET /wp-admin/plugins.php
# -- spread across many source IPs at roughly 4 req/min per IP, each hit
# burning a PHP-FPM/lsphp worker. Two sites absorbed 1,243 resulting 503s
# in nine hours as their pools saturated.
#
# IDENTITY, NOT RATE. This is the one rule in this file that tracks
# nothing and has no stick-table counter or threshold. Every rate-based
# control above (the generic limits, wp_bruteforce, xmlrpc_bruteforce) is
# per-IP, and per-IP rate is exactly what this attack is engineered to
# stay under: ~4 req/min from any single IP is indistinguishable from a
# slow human, and the source set is large enough that no threshold can be
# lowered to catch it without also catching real visitors. There is also
# no free stick-table slot left to try anyway -- sc0/sc1/sc2 are already
# used above and HAProxy's default tune.stick-counters is 3, so a fourth
# tracked counter is not an option here. DO NOT "simplify" this into a
# rate/threshold rule later: the whole point is that a threshold cannot
# see this traffic. Instead we gate on identity -- a real logged-in
# WordPress user always carries a wordpress_logged_in_* cookie (the same
# ACL the wp2shell block above already declares), and an unauthenticated
# request to a wp-admin page has no legitimate reason to boot PHP at all.
#
# 302, not 403. WordPress itself redirects an unauthenticated /wp-admin/
# request to wp-login.php, so replicating that at the edge means an admin
# whose session merely expired lands on the normal login screen instead
# of an error page -- we are not trading a bot problem for a support
# ticket. Bots get a cheap redirect they ignore.
#
# path_reg, not path_beg. A subdirectory install at /blog/wp-admin/ slips
# past a prefix match; path_reg with an optional leading "/" catches both
# root and subdirectory installs, same reasoning as the wp-login and
# xmlrpc path_end rules above.
#
# regsub rewrites the redirect target itself, so /blog/wp-admin/x.php
# redirects to /blog/wp-login.php rather than 404ing at the site root.
#
# DO NOT write regsub's regex argument with a capturing group / literal
# parentheses, e.g. regsub((^|/)wp-admin/.*,\1wp-login.php) -- neither
# inlined into the redirect's `location` nor in a standalone set-var.
# HAProxy 3.0.11's converter-argument parser counts parens to find the
# end of the regsub(...) call itself, so the *inner* "(^|/)" grouping
# parens are misread as closing the outer call early -- it does not
# matter whether the argument is quoted ("...": still fails) or the
# parens are backslash-escaped (\(...\): still fails). Every such form
# was verified against real HAProxy 3.0.11-1+deb13u3 and all produce the
# same ALERT: "invalid arg 2 in converter 'regsub' : missing arguments
# (got 1/2)". This is a converter-argument-parsing limitation, not a
# log-format/`%[...]` issue -- the identical failure reproduces in a
# plain set-var (outside any log-format string), which rules out the
# `location` value's log-format context as the cause.
#
# The fix sidesteps groups/backreferences entirely: HTTP paths always
# start with "/", so the leading "(^|/)" alternation is redundant --
# matching the literal substring "/wp-admin/" (both slashes, no group)
# is sufficient to anchor to a real path segment (a false match like
# "/somewp-admin/" doesn't contain "/wp-admin/" as a substring, since
# there's no "/" directly before "wp-admin"). No backreference is
# needed either: regsub only replaces the matched substring, so
# replacing "/wp-admin/.*" with a literal "/wp-login.php" leaves
# whatever precedes it (the subdirectory-install prefix, if any)
# untouched. Computed in its own set-var so it is a plain sample
# expression, not something baked into the redirect's log-format
# string. Behaviorally verified live against real HAProxy 3.0.11:
# /wp-admin/edit.php -> /wp-login.php and
# /blog/wp-admin/plugins.php -> /blog/wp-login.php, both with
# redirect_to preserved. See
# .superpowers/sdd/2026-08-14-wpadmin-edge-gate/task-3b-report.md.
#
# redirect_to carries the path only (%[path,url_enc]), not the query
# string -- deliberate, see design spec. An admin bounced off
# post.php?post=123&action=edit lands back on a blank post.php rather
# than that exact post. Capturing the full URI needs capture.req.uri
# (extra config, and the captured value is length-capped) for a benefit
# that only matters on session expiry, so it was not worth it.
#
# THE ALLOWLIST IS MEASURED, NOT GUESSED -- taken from actual fleet
# traffic returning 200 on /wp-admin/*. admin-ajax.php and admin-post.php
# are the standard front-end AJAX/form-handler endpoints real themes and
# plugins call while logged out. Critically, wp-login.php loads its OWN
# css/js FROM /wp-admin/ (load-styles.php, load-scripts.php, and the
# static wp_admin_asset dirs below) -- miss those and every login page on
# the fleet renders unstyled with a broken password-strength meter, while
# wp-login.php itself still returns 200, making it a silent regression
# that "looks like" the gate is working.
#
# Each allowlist entry is anchored to /wp-admin/<file>, not a bare
# filename suffix. A bare `path_end /admin-ajax.php` also matches
# /wp-admin/evil/admin-ajax.php -- which ALSO matches wp_admin_path
# (path_reg only requires /wp-admin/ to appear somewhere), so an
# attacker-inserted path segment would sail through this allowlist
# ungated and boot full WordPress, exactly the resource exhaustion this
# gate exists to stop. Anchoring still covers subdirectory installs via
# suffix matching (/blog/wp-admin/admin-ajax.php ends with
# /wp-admin/admin-ajax.php) while rejecting an inserted directory.
#
# install.php is DELIBERATELY NOT allowlisted. It is legitimately
# reachable without a cookie during a fresh install, but it is also a
# standing scanner target and a real takeover vector on a site that was
# half-installed and then abandoned. Anyone genuinely installing uses the
# per-site exempt-list opt-out below instead.
#
# Honors the same whitelist as every other rule in this frontend
# (RFC1918 / trusted_ips.list / trusted_ips.map), and a per-site opt-out
# via /etc/haproxy/wpadmin_gate_exempt.list (operator-managed, seeded
# empty by start-up.sh) for sites where a plugin legitimately serves
# unauthenticated visitors from a /wp-admin/ URL outside this allowlist.
# EVERY ACL BELOW MATCHES THE NORMALISED PATH. The normalize-uri chain at
# the top of this frontend has already merged duplicate slashes, resolved
# "." / ".." segments (including percent-encoded ones) and decoded
# unreserved escapes by the time these run, so these patterns only have to
# describe the ONE canonical spelling the backend will resolve -- they do
# not have to anticipate every encoding of it. That is the whole point of
# the normalisation block; do not "harden" these regexes by re-adding
# encoding variants, fix the normalisation instead.
#
# wp_admin_safe_path guards against an OPEN REDIRECT this gate would
# otherwise introduce. The redirect target below is built by rewriting
# `path` with regsub -- regsub only replaces the matched substring, so
# everything BEFORE the matched "/wp-admin/" survives untouched in the
# output. `path` is not guaranteed to be a clean site-relative string;
# three concrete requests turn that survival into an off-site
# `Location:` header:
# //evil.example.com/wp-admin/x.php -> //evil.example.com/wp-login.php
# (protocol-relative -- browsers resolve "//host/path" to
# "https://host/path", so this redirects off-site with no scheme
# needed). NOW NEUTRALISED UPSTREAM: path-merge-slashes rewrites this
# to /evil.example.com/wp-admin/x.php before any ACL sees it, so the
# Location becomes the same-origin /evil.example.com/wp-login.php.
# Verified live.
# /\evil.example.com/wp-admin/x.php -> /\evil.example.com/wp-login.php
# (browsers normalise a leading "/\" the same as "//"). STILL LIVE
# after normalisation -- a backslash is not a slash, so no normalizer
# touches it. This ACL is the only thing that stops it.
# https://evil.example.com/wp-admin/x.php -> https://evil.example.com/wp-login.php
# (RFC 7230 absolute-form request targets can make HAProxy's `path`
# fetch return a full URI, not just the path component). HAProxy's own
# H1 parser answers 400 on this frontend; this ACL is the backstop.
# So wp_admin_safe_path is NOT redundant with the normalisation and must
# not be deleted as such -- one of its three vectors survives normalisation
# untouched.
#
# It is used TWO ways, and the pair matters:
# - as a POSITIVE condition on the redirect, so a pathological path can
# never produce a `Location:` header at all; and
# - as an explicit deny, so such a path is not merely un-redirected.
# The deny is what closes the failure mode the positive-condition form
# introduced on its own: "not redirected" used to mean "falls through to
# the backend UNGATED", i.e. the exact PHP-booting request this gate
# exists to stop, reachable by prefixing "//" (that specific spelling is
# now normalised away, but "/\" is not). Post-normalisation the only
# paths that reach the deny are "/\..." ones, which cannot resolve to a
# real file on any tier, so nothing legitimate is denied.
# -i (case-insensitive) and the matching ",i" flag on the regsub below are
# a PAIR -- adding one without the other produces an infinite redirect
# loop, because a case-sensitive regsub finds no "/wp-admin/" in
# "/WP-ADMIN/plugins.php", returns `path` UNCHANGED, and the Location then
# points at the request's own URL. Verified live that the pair is correct:
# /WP-ADMIN/plugins.php -> /wp-login.php and /blog/WP-Admin/plugins.php ->
# /blog/wp-login.php.
#
# On this fleet's Linux backends /WP-ADMIN/plugins.php 404s without booting
# PHP, so this is hardening rather than a live-bypass fix; it matters if a
# docroot ever sits on a case-insensitive mount, where that same request
# WOULD boot PHP. The cost is that a site with a real directory literally
# named e.g. /docs/WP-Admin/ now gets gated -- the same false positive the
# lowercase pattern already has, which is what the per-site exempt list
# exists to resolve.
acl wp_admin_path path_reg -i (^|/)wp-admin/
# Four literal backslashes here is NOT a typo. HAProxy's config-line word
# parser treats backslash as its OWN escape character before the value
# ever reaches the regex engine: "\\" (two backslashes) in the config
# collapses to one literal backslash by the time PCRE compiles it, which
# leaves an unterminated character class ("[^/\]") and fails with
# "missing terminating ] for character class" -- verified against real
# HAProxy 3.0.11. Four backslashes ("\\\\") collapse to two ("\\"),
# which PCRE then reads as a single escaped-backslash class member --
# the intended "reject a literal backslash" semantics.
acl wp_admin_safe_path path_reg ^/[^/\\\\]
acl wp_admin_allowed path_end /wp-admin/admin-ajax.php /wp-admin/admin-post.php /wp-admin/load-styles.php /wp-admin/load-scripts.php
# The (?!.*\.php) lookahead is DEFENCE IN DEPTH, not the primary fix. This
# ACL grants an un-gated bypass to everything under wp-admin/css|js|images,
# and it used to anchor its prefix but not its suffix, so
# /wp-admin/css/../plugins.php took the bypass and the backend then
# resolved ".." and booted plugins.php. path-strip-dotdot now rewrites that
# to /wp-admin/plugins.php before this ACL runs, which is the real fix; the
# lookahead additionally makes the bypass structurally incapable of
# covering a PHP entrypoint even if a future encoding trick survives
# normalisation. It excludes ".php" ONLY -- no static asset contains that
# substring, so it cannot cause the silent "login page renders unstyled"
# regression that an extension allowlist would risk. Requires PCRE2, which
# both the Debian (deployed) and Alpine haproxy builds have (+PCRE2).
acl wp_admin_asset path_reg (^|/)wp-admin/(css|js|images)/(?!.*\.php).*$
acl wp_gate_exempt hdr(host),lower -f /etc/haproxy/wpadmin_gate_exempt.list
# ENCODED SEPARATOR. percent-decode-unreserved deliberately does NOT decode
# %2F -- "/" is a reserved character, and decoding it in the normalizer
# would change the path's structure (it would invent new segments), which
# is precisely why HAProxy refuses to. But OpenLiteSpeed DOES decode it and
# then serves the file: /wp-admin%2Fplugins.php was measured returning 302
# from a real WordPress site on the OLS tier, i.e. full PHP boot, while
# matching none of the ACLs above. Apache returns 404 for the same request
# (AllowEncodedSlashes Off), so this is an OLS-tier defect -- and OLS is the
# tier currently saturating.
#
# DENY, not "treat it as a wp-admin path and redirect". Two reasons:
# 1. The redirect target is computed by regsub(/wp-admin/.*) which finds
# no "/wp-admin/" in "/wp-admin%2Fplugins.php", so `path` would come
# back UNCHANGED and the Location would point at the request's own
# URL -- an infinite redirect loop, not a gate.
# 2. Nothing legitimate emits it. A path segment cannot contain a literal
# "/", so %2F inside a path is always either a probe or a proxy-
# confusion attempt, and the Apache tier has been 404ing it all along,
# so no site on the fleet can depend on it.
#
# SCOPED to paths that mention wp-admin, not all paths. A blanket "deny any
# %2F in any path" would also hit REST/API-style routes on non-WordPress
# customer apps that legitimately pass an encoded slash inside a path
# parameter. Scoping keeps the blast radius inside the attack surface this
# gate owns.
#
# Matching is on the SUBSTRING, not an anchored pattern, on purpose:
# /blog%2Fwp-admin/plugins.php hides the separator BEFORE "wp-admin", where
# an anchored (^|/)wp-admin/ never matches, and OLS still resolves it to
# /blog/wp-admin/plugins.php. Substring matching catches the separator
# wherever it is. percent-to-uppercase has already folded %2f into %2F;
# the -i is belt and braces so this rule stands on its own if the
# normalizer is ever reordered.
#
# %5C (encoded backslash) is denied on the same terms. On this fleet's
# Linux backends a backslash is an ordinary filename character, so
# /wp-admin%5Cplugins.php 404s rather than booting PHP -- measured, it is
# not a live bypass today. It is included because it is the same
# encoded-separator trick against a backend that happens to treat "\" as
# one, it costs nothing, and no legitimate path contains it.
acl wp_admin_word path -i -m sub wp-admin
acl path_has_encoded_sep path -i -m sub %2f %5c
http-request deny deny_status 403 if wp_admin_word path_has_encoded_sep !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted
http-request deny deny_status 403 if wp_admin_path !wp_admin_safe_path !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted
http-request set-var(txn.wp_login_url) path,regsub(/wp-admin/.*,/wp-login.php,i) if wp_admin_path
http-request redirect code 302 location %[var(txn.wp_login_url)]?redirect_to=%[path,url_enc] if wp_admin_path wp_admin_safe_path !wp_admin_allowed !wp_admin_asset !has_wp_logged_in !wp_gate_exempt !is_local !is_trusted_ip !is_whitelisted
# IP blocking using map file (manual blocks only)
# Map file format: /etc/haproxy/blocked_ips.map contains "<ip_or_cidr> 1" per line
# Runtime updates: echo "add map #0 IP_ADDRESS 1" | socat stdio /var/run/haproxy.sock
wpadmin_gate_exempt.list
+16
View File
@@ -0,0 +1,16 @@
# Per-site opt-out from the WordPress admin edge gate.
#
# Hostnames listed here are EXEMPT: unauthenticated /wp-admin/* requests for
# these sites pass through to PHP instead of being redirected to wp-login.php.
# One hostname per line, lowercase. Matched against the Host header.
#
# Referenced by templates/hap_listener.tpl:
# acl wp_gate_exempt hdr(host),lower -f /etc/haproxy/wpadmin_gate_exempt.list
#
# Add a site here when a plugin legitimately serves unauthenticated visitors
# from a /wp-admin/ URL that is not in the rule's allowlist. Symptom: "my
# plugin's admin page redirects to login".
#
# Do NOT commit real customer domains — this repo is mirrored publicly. Add
# entries directly on the server; the file lives in the /etc/haproxy named
# volume and persists across container recreates.