shadowdaoandClaude Opus 5
22dab685d0
Every path that refreshed a combined certificate did
with open(combined_path, 'w') as combined: # TRUNCATES
subprocess.run(['cat', cert, key], stdout=combined) # rc ignored
where combined_path is the bundle HAProxy is currently serving. The live file
was emptied before any source material had been read and the cat status was
never checked, so an unreadable source, a zero-length privkey, a full disk or a
killed container left a truncated or key-less PEM in place. HAProxy loads
/etc/haproxy/certs as a directory, so one unusable file there fails the whole
ssl bind - HTTPS down for every site on the host, and unlike a bad haproxy.cfg
it is not recoverable by config rollback.
The bundle endpoint compounded it: superseded .pem files were unlinked and
their lineages `certbot delete`d before anything had checked the replacement,
destroying both copies of a working certificate. Recovery there means fresh,
rate-limited ACME orders.
Bundles are now assembled in a staging directory beside the crt directory (never
inside it - HAProxy would try to load a temp file), validated there, and swapped
in with os.replace(). Validation is mandatory structural checks in pure Python
plus a best-effort openssl key/leaf pairing check; a missing openssl warns
loudly and does not silently pass. The previous bundle is copied to
/etc/haproxy/cert-backups first. Superseded certs are moved aside rather than
deleted, and `certbot delete` runs only after HAProxy has reloaded onto the
replacement. The same guarantees are implemented for the cron/renewal shell
path in scripts/cert-publish-lib.sh, which also gates the reload on `haproxy -c`.
Reuses write_config_atomically() from the config-rollback fix (extended with
staging_dir/validate) rather than adding a second atomic writer.
Also heals a zero-byte QUIC cluster-secret file, which previously made
get_or_create_cluster_secret() return '' forever.
Tests: scripts/test-cert-write-safety.py (17) and scripts/test-cert-scripts.py
(32) in the existing stdlib-unittest, stub-binary convention. Both bug classes
reproduce against the pre-fix tree via HAPROXY_MANAGER_DIR.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
637 lines
28 KiB
Python
Executable File
637 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Regression tests for the certificate publication shell scripts.
|
|
|
|
Why this file exists
|
|
--------------------
|
|
renew-certificates.sh and sync-certificates.sh published a bundle with
|
|
|
|
cat "$CERT_FILE" "$KEY_FILE" > "$COMBINED_FILE"
|
|
|
|
where $COMBINED_FILE is the pem HAProxy is serving *right now*. The shell
|
|
truncates the destination when it opens the redirect, before cat runs, so any
|
|
failure after that point - unreadable source key, ENOSPC, container killed
|
|
mid-write - left a zero-length or key-less pem behind. The exit status of cat
|
|
was checked, but by then the live file was already destroyed.
|
|
|
|
HAProxy loads $SSL_CERTS_DIR as a DIRECTORY (`bind :443 ssl crt /etc/haproxy/
|
|
certs`) and tries to load every file in it, so a single unloadable file fails
|
|
the whole bind: HTTPS down for every customer on the host.
|
|
|
|
The fix (scripts/cert-publish-lib.sh) assembles into a sibling staging dir,
|
|
validates, backs up the outgoing bundle into a sibling backup dir, and renames
|
|
into place. These tests pin the observable guarantees:
|
|
|
|
* a successful publish replaces the live pem and archives the old one;
|
|
* a FAILED publish leaves the previous, still-valid pem byte-for-byte intact;
|
|
* nothing that is not a final *.pem ever appears in the certs directory;
|
|
* HAProxy is not reloaded when `haproxy -c` rejects the configuration.
|
|
|
|
Running
|
|
-------
|
|
python3 scripts/test-cert-scripts.py # tests the repo checkout
|
|
HAPROXY_MANAGER_DIR=/some/other/tree \
|
|
python3 scripts/test-cert-scripts.py # tests another tree
|
|
|
|
Self-contained stdlib unittest - no pytest, no venv, no bats, and nothing is
|
|
imported from the application. The scripts are driven as subprocesses with
|
|
every path they touch redirected by environment variable (SSL_CERTS_DIR,
|
|
LETSENCRYPT_LIVE_DIR, CERT_STAGING_DIR, CERT_BACKUP_DIR, LOG_FILE,
|
|
ERROR_LOG_FILE, HAPROXY_CONFIG) and stub `certbot`, `socat` and `haproxy`
|
|
binaries on PATH.
|
|
|
|
The certificate material below is a real self-signed test certificate with its
|
|
matching key (plus a second, unrelated key for the mismatch case), embedded as
|
|
constants so the tests need no openssl to *create* material. The one test that
|
|
needs openssl to *verify* pairing skips itself if the binary is absent.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
|
|
MODULE_DIR = os.path.abspath(
|
|
os.environ.get('HAPROXY_MANAGER_DIR',
|
|
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
|
)
|
|
SCRIPTS_DIR = os.path.join(MODULE_DIR, 'scripts')
|
|
LIB = os.path.join(SCRIPTS_DIR, 'cert-publish-lib.sh')
|
|
|
|
BROKEN_TOKEN = '__BROKEN__'
|
|
DOMAIN = 'test.example.com'
|
|
|
|
# --- test key material -------------------------------------------------------
|
|
# openssl req -x509 -newkey rsa:2048 -keyout key1 -out cert1 -days 3650 -nodes \
|
|
# -subj /CN=test.example.com
|
|
TEST_CERT = """\
|
|
-----BEGIN CERTIFICATE-----
|
|
MIIDFzCCAf+gAwIBAgIUeaz/lNOESOTHsB3Y97+Xja7Fy4gwDQYJKoZIhvcNAQEL
|
|
BQAwGzEZMBcGA1UEAwwQdGVzdC5leGFtcGxlLmNvbTAeFw0yNjA4MDYxNTQyMTda
|
|
Fw0zNjA4MDMxNTQyMTdaMBsxGTAXBgNVBAMMEHRlc3QuZXhhbXBsZS5jb20wggEi
|
|
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDF5GL7Gjn+UnPFy5sqP2k4XHth
|
|
mkWFZj+mjK6cDhbBXYt60NrwVdrrgOFydMC75VeUceFxG/5GD7wrXZP23xzbnWKm
|
|
7FxfOSmr4y+1rVEZwi8IeWEz3W6C6y5rjZsCI+pBgdna+aJSpTQZHPfDpNtQm5vl
|
|
enj5BfizYixinORxm9kvXMGXV+Cw1CJkqB3mzScwWt40EtQoVxekebf8B7i4ZHyx
|
|
xT6/xwF+WY8OliZkY1pdqncoTLUAYcaE/HR/ojJKmSVIq1GswZE/y3E56LIwq+wJ
|
|
eGbgH46a+86z+VO2UX1jbad1kWKBCsRoOpaybZDEWAYTOqahW7kOH7umTUsRAgMB
|
|
AAGjUzBRMB0GA1UdDgQWBBTIqa3BNEjjcxkhXqnRwInrLM9yijAfBgNVHSMEGDAW
|
|
gBTIqa3BNEjjcxkhXqnRwInrLM9yijAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
|
|
DQEBCwUAA4IBAQB/lYzXb5PI3magMz/IXmwTsMCrVSdaUYEIKLEJggmbGxqpwO1a
|
|
iYagWZ/5H3B9KDvNQA+L4FkMJ726ZkdGEH/vkwvTAuhwU2NSWcbRJ8DK5u3Q4rnJ
|
|
VswPcW5njUF9mQq0NPX/PMCeOoFDEI8+RrgQZxtHhopwuKOgVA6HRBINKdEZJlrp
|
|
oLLQrHDNVLMYTclNHG6kBg0lOHUV31TgkJQ8kMgtq0WQX7RseKR10QKgN5iOBomU
|
|
3+y713Ibpac5B1zw5l3LjE/59xFteFbDENr2+A5VGhVNVZC+bs+YTYTzeAsGB0e3
|
|
MQ+XJMJq3kaJmQ+QcTrRaKMtoMz2h1AIRbLd
|
|
-----END CERTIFICATE-----
|
|
"""
|
|
|
|
# The private key that matches TEST_CERT.
|
|
TEST_KEY = """\
|
|
-----BEGIN PRIVATE KEY-----
|
|
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDF5GL7Gjn+UnPF
|
|
y5sqP2k4XHthmkWFZj+mjK6cDhbBXYt60NrwVdrrgOFydMC75VeUceFxG/5GD7wr
|
|
XZP23xzbnWKm7FxfOSmr4y+1rVEZwi8IeWEz3W6C6y5rjZsCI+pBgdna+aJSpTQZ
|
|
HPfDpNtQm5vlenj5BfizYixinORxm9kvXMGXV+Cw1CJkqB3mzScwWt40EtQoVxek
|
|
ebf8B7i4ZHyxxT6/xwF+WY8OliZkY1pdqncoTLUAYcaE/HR/ojJKmSVIq1GswZE/
|
|
y3E56LIwq+wJeGbgH46a+86z+VO2UX1jbad1kWKBCsRoOpaybZDEWAYTOqahW7kO
|
|
H7umTUsRAgMBAAECggEAA9SnFFqGfR1yO4XUlfmmgyZqJoJmnl2TdZlDT4bHyrwx
|
|
dSIKHO0iiNzEsHMhYHnA62EVdruunUNUdofwE28v9zHZnSbV5mt8OqUyERue5mdf
|
|
gvPbjXYXu63LBx61fZH9qME3WwFqUpx7UNGiW62LJ8ktWEC5ywNCNFG+D3YfR3Iu
|
|
v3PdIFXwkpJMaXO+42JSoSVoiqxlONqNiqcdQj64iYgCYdNxcgdLy+mHGrf1hAZs
|
|
dom2Jp0oEM4ZdOQu3Z6uyEyPsECiz1PdQjzagaEfPWtKCQk0kY8DDCn5X2xQif9w
|
|
3xeaISqho7bQgSo4IEgWHTP6V0v7brTP7VcK+gyMyQKBgQD15FrE4XAL89NBQ6iX
|
|
m2l6quZv5tIUtreuYxsOpFIMU22zfsRZOvA7sJR3ufOJ+b7tFHuJ8DAJUpK08Vvg
|
|
A930/LW9wUsY42d54XKIO/8DTsIrmjppoGdshs3axJQkqfN0zQkhqfbwaXRJ9X2k
|
|
Fvax+5jftaIaw0hQdWLnfTmGmQKBgQDOBue89D8THYl/LvDGg6WoVyK3ljBMK2s3
|
|
4BljeZCJdWAjtido13Mltubc6YVHScmVoIZKTmx+fCjdQ3y1t92vZZSBeLsXhfFr
|
|
N+IOGZu3ZmJu64x3OukSYQ7x6agi5yP3+7k0siZgxOMXJRQDYZUcHxIHDrSGLeLZ
|
|
sj7LnbvLOQKBgE8murE1gEPYsOAJT3O96y45ZQQQYP+Z8XaJIGSOMHsXP/DPlZTD
|
|
jCEqrh/8E5EOe48FUN8OGehmVCM6rkBl/kSmNDpoxiu0x9JL5/pClcwSxh4S/0qQ
|
|
/7nHiuwo6ycCLgQjHBViCMNKrsw/4bm4SqDwRD1+0jebNOPxZWzuul3BAoGAZij0
|
|
ZjSyxhbCZEdxau5CiYvTkjct8cch3k4IKNRRwGdsaajcN9eFqHDeXzKIPQYwqDo1
|
|
/MiQcdO9K6JYR39JtLxo/B5Sn2JyiJjoRdea6EEjlB7GwyR6B/wKvhf/oHb+1euD
|
|
NccU0q0ucf6XwulzV8NsXAWFrHc6YnpJOwwW37kCgYAkEyjUc73jImiTyf/IXVOD
|
|
UHlRXZPvwtZUuPGe4RI0Gds97tKnvXnvFsPIRCOGfVzZ8z79DGiQ8TR2a0hgZec1
|
|
Mo3J2dCjlv4Q6ACjHkCA1cmi13OHUPnpaeesrOk+SpEVJfR2k4qRWH4z3oxbPO/q
|
|
TDFnjWHSgjkDMrxVHZ8wPg==
|
|
-----END PRIVATE KEY-----
|
|
"""
|
|
|
|
# A perfectly valid RSA key that has nothing to do with TEST_CERT.
|
|
UNRELATED_KEY = """\
|
|
-----BEGIN PRIVATE KEY-----
|
|
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDE7mosrsdBhdn1
|
|
ZIErYmMuPU69ws51JdyqwtRZlV2uLLby0dOfJpFKZPuBPEqTqDimd9N4FSL0CGjG
|
|
zXG23RYd9AbdSoorFORrUxeiPzFnbz4v38srGHckOS9ozKAbPLACUEjWMX1NwAZ+
|
|
wGlSz3cWYTWVztFCUIxpvLVR87PSpTnpdCXIj0EABOc6WoLBLE+v2knZOa63LKg6
|
|
GryInD43CnWFBKpH0gdgWqh+ie3NFMumLR8M3lZq2Mk0EFgVWrxPnJobYvOmavNp
|
|
NBR3ZugR4X57c1YFyLkYXQIdhxuYV1ZAY6NgmAIsaiCKFqdksF5AJSV7FZNBM43r
|
|
33ae/R5hAgMBAAECggEAIgNYsL9+OEWtU8ooWi0r2q5plXJaVNb1gkPUx+U5sS3V
|
|
amKNxbj0UrBW1Scr7U1afXwIOP8TkqkKKb4NpCsS2RkO/3USoKbC3fuTwyjdeFM5
|
|
Hy0sytR2rXm4A8aF57ZnYvrpXZ9eGEnwhT9n4Y7mL2YaSnXWZDkDy3Z1rcIk/p5P
|
|
jUo4UQyzD/9Gab1stcBbGv+66B3mlVdRVJor5+tGn6zmr4TjvqupAexuXd+Q6+1L
|
|
OG4e2bAUmuOVOa+w8Xo4vwkiXSDLVEsl1z1x4Bcrv61bIg0rbZAeqcZ4EUPyrEyR
|
|
RpaaOLAiwdzjOj5pQvHAF2q/++IlY6xPhotTr11tvQKBgQD37C2UQRGRo622+2kx
|
|
qgsdA6jPM9vCE9S4aS4qaOj8XSMMCu6bQW5lU48DaONvXxDmT6x9IrgR3QN1Iz/t
|
|
17kUCCghuviEP1RjnahBRQqZR38KdDzKhaWt9jgdjCun9MzUXIaQRTFlFGA1z1pz
|
|
apZ9a8/ehYPSk4pv9h06/B3IjQKBgQDLWPBuhj203QOmwVF1v1k4yyuJ7UBqtRaY
|
|
I9jMW93sB2hPs6Se10UroQHlF95IHeRXvNKH/UXuAILIJMN8oTI4uU7t60shJHvI
|
|
o14RzEZoUQjES7BBVhePglndJmYIKoKKAX5lSFe9Lk0ei9B+1Dle5Q+9ZeP09cp4
|
|
vVcGvOgqJQKBgQCtxe1srO8TlhZ821uwY+/GNnpsQX0XW68OUyr4rvAfc2jNWBxG
|
|
1mX6v8bOLQa9WXUO+Wl9jIhYfQGfaUW2AC7Jy63VdqgaigksiaUVmr8DEQoK2c6C
|
|
ZYrrlFlg3I78+qlXcEMhfF5S6yVEkkJkA6HX52mcHxl2z9OJBokWfwChQQKBgAzH
|
|
xzy7FS/D4FHfvpXu89Wc91yQ28aZIRVo01xsvbLy+DxiJwuQrhlC4lKawG656jsV
|
|
dAn2AiomQBICNYMkwnpMM0jCzBMGLv16PxRRSW+PAEUOGMLSfWKYp7s9iZYjzdaM
|
|
p3wIIvOR8GjmErGV9xEexnF58OzZceNKyyhyQQk9AoGAbvjJvYLOhQpAbbqErNNv
|
|
LK1P+TngKpukRHXjiUpPVEGNhN6krBBJCBWzY7ucrIy6jz8UBy7SbITBy7qIKhxZ
|
|
PVP7WVATMWEeW1AfdBfCYDI4jFKAD8SLECby45nRBuBllYdQnW1gBzLcCulCwB+w
|
|
FV0RvuQPDYkqsx8ibqpSv7c=
|
|
-----END PRIVATE KEY-----
|
|
"""
|
|
|
|
# The bundle already on disk when a run starts. Same key material, plus a
|
|
# trailing marker so "the live file was replaced" and "the old file was
|
|
# archived" can be told apart byte-for-byte.
|
|
PREVIOUS_BUNDLE = TEST_CERT + TEST_KEY + '# previous bundle\n'
|
|
NEW_BUNDLE = TEST_CERT + TEST_KEY
|
|
|
|
# --- stub binaries -----------------------------------------------------------
|
|
STUB_CERTBOT = """\
|
|
#!/bin/sh
|
|
# Test stub for certbot: pretend there was nothing to renew.
|
|
echo "No renewals were attempted."
|
|
exit 0
|
|
"""
|
|
|
|
STUB_SOCAT = """\
|
|
#!/bin/sh
|
|
# Test stub for socat: record that a reload was attempted, and what was sent.
|
|
{ printf 'socat %s <<' "$*"; cat; printf '>>\\n'; } >> "$SOCAT_LOG"
|
|
exit 0
|
|
"""
|
|
|
|
STUB_HAPROXY = """\
|
|
#!/bin/sh
|
|
# Test stub for the haproxy binary: `haproxy -c -f FILE` rejects any config
|
|
# containing %(token)s, which is how the tests inject an invalid config.
|
|
cfg=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in -f) cfg="$2"; shift ;; esac
|
|
shift
|
|
done
|
|
if [ -n "$cfg" ] && grep -q '%(token)s' "$cfg" 2>/dev/null; then
|
|
echo "[ALERT] parsing [$cfg:1] : unknown keyword '%(token)s'" >&2
|
|
exit 1
|
|
fi
|
|
exit 0
|
|
""" % {'token': BROKEN_TOKEN}
|
|
|
|
GOOD_HAPROXY_CFG = textwrap.dedent("""\
|
|
global
|
|
daemon
|
|
defaults
|
|
mode http
|
|
frontend fe
|
|
bind 0.0.0.0:443 ssl crt /etc/haproxy/certs
|
|
""")
|
|
|
|
|
|
def write(path, content, mode=None):
|
|
with open(path, 'w') as fh:
|
|
fh.write(content)
|
|
if mode is not None:
|
|
os.chmod(path, mode)
|
|
return path
|
|
|
|
|
|
def read(path):
|
|
with open(path) as fh:
|
|
return fh.read()
|
|
|
|
|
|
class CertScriptFixture(unittest.TestCase):
|
|
"""An isolated fake /etc/haproxy + /etc/letsencrypt plus stub binaries."""
|
|
|
|
def setUp(self):
|
|
self.tmp = tempfile.mkdtemp(prefix='haproxy-cert-test-')
|
|
self.addCleanup(self._cleanup_tmp)
|
|
|
|
self.bindir = os.path.join(self.tmp, 'bin')
|
|
os.makedirs(self.bindir)
|
|
write(os.path.join(self.bindir, 'certbot'), STUB_CERTBOT, 0o755)
|
|
write(os.path.join(self.bindir, 'socat'), STUB_SOCAT, 0o755)
|
|
write(os.path.join(self.bindir, 'haproxy'), STUB_HAPROXY, 0o755)
|
|
self.socat_log = os.path.join(self.tmp, 'socat-invocations.log')
|
|
|
|
# Mirrors the real layout: certs dir, with staging/backups as SIBLINGS.
|
|
self.haproxy_dir = os.path.join(self.tmp, 'etc', 'haproxy')
|
|
self.certs_dir = os.path.join(self.haproxy_dir, 'certs')
|
|
self.staging_dir = os.path.join(self.haproxy_dir, 'cert-staging')
|
|
self.backup_dir = os.path.join(self.haproxy_dir, 'cert-backups')
|
|
os.makedirs(self.certs_dir)
|
|
|
|
self.le_live = os.path.join(self.tmp, 'etc', 'letsencrypt', 'live')
|
|
self.domain_dir = os.path.join(self.le_live, DOMAIN)
|
|
os.makedirs(self.domain_dir)
|
|
self.src_cert = write(os.path.join(self.domain_dir, 'fullchain.pem'), TEST_CERT)
|
|
self.src_key = write(os.path.join(self.domain_dir, 'privkey.pem'), TEST_KEY)
|
|
|
|
self.live_pem = os.path.join(self.certs_dir, DOMAIN + '.pem')
|
|
self.haproxy_cfg = write(os.path.join(self.haproxy_dir, 'haproxy.cfg'),
|
|
GOOD_HAPROXY_CFG)
|
|
self.log_file = os.path.join(self.tmp, 'haproxy-manager.log')
|
|
self.error_log = os.path.join(self.tmp, 'haproxy-manager-errors.log')
|
|
|
|
def _cleanup_tmp(self):
|
|
# A test may have chmod 000'd a fixture file.
|
|
for root, dirs, files in os.walk(self.tmp):
|
|
for name in files:
|
|
try:
|
|
os.chmod(os.path.join(root, name), 0o600)
|
|
except OSError:
|
|
pass
|
|
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
|
|
# -- helpers ---------------------------------------------------------
|
|
def env(self, **overrides):
|
|
env = dict(os.environ)
|
|
env.update({
|
|
'PATH': self.bindir + os.pathsep + os.environ['PATH'],
|
|
'SSL_CERTS_DIR': self.certs_dir,
|
|
'LETSENCRYPT_LIVE_DIR': self.le_live,
|
|
'CERT_STAGING_DIR': self.staging_dir,
|
|
'CERT_BACKUP_DIR': self.backup_dir,
|
|
'LOG_FILE': self.log_file,
|
|
'ERROR_LOG_FILE': self.error_log,
|
|
'HAPROXY_CONFIG': self.haproxy_cfg,
|
|
'SOCAT_LOG': self.socat_log,
|
|
})
|
|
env.update(overrides)
|
|
return env
|
|
|
|
def run_script(self, name, **env_overrides):
|
|
path = os.path.join(SCRIPTS_DIR, name)
|
|
self.assertTrue(os.path.exists(path), f'{path} does not exist')
|
|
return subprocess.run(['bash', path], env=self.env(**env_overrides),
|
|
capture_output=True, text=True, timeout=120)
|
|
|
|
def bundle_is_valid(self, path):
|
|
"""Ask the shipped library whether HAProxy could use this bundle."""
|
|
return subprocess.run(
|
|
['bash', '-c', '. "$1"; cert_bundle_valid "$2"', '_', LIB, path],
|
|
env=self.env(), capture_output=True, text=True).returncode == 0
|
|
|
|
def reload_attempted(self):
|
|
return os.path.exists(self.socat_log) and os.path.getsize(self.socat_log) > 0
|
|
|
|
def logs(self):
|
|
text = ''
|
|
for path in (self.log_file, self.error_log):
|
|
if os.path.exists(path):
|
|
text += read(path)
|
|
return text
|
|
|
|
def seed_previous_bundle(self, content=PREVIOUS_BUNDLE):
|
|
return write(self.live_pem, content)
|
|
|
|
def assert_certs_dir_is_clean(self):
|
|
"""HAProxy loads this directory wholesale: only final *.pem may be here."""
|
|
entries = sorted(os.listdir(self.certs_dir))
|
|
strays = [e for e in entries if not e.endswith('.pem')]
|
|
self.assertEqual(strays, [],
|
|
f'non-.pem files left in the certs directory HAProxy '
|
|
f'loads wholesale: {strays} (dir: {entries})')
|
|
|
|
def assert_no_staging_leftovers(self):
|
|
if os.path.isdir(self.staging_dir):
|
|
self.assertEqual(sorted(os.listdir(self.staging_dir)), [],
|
|
'staging file was not cleaned up')
|
|
|
|
|
|
class CertScriptBehaviour:
|
|
"""Behaviour shared by renew-certificates.sh and sync-certificates.sh.
|
|
|
|
A mixin rather than a TestCase so the cases are collected once per concrete
|
|
script, not a third time for the base class.
|
|
"""
|
|
|
|
SCRIPT = None
|
|
|
|
def run_it(self, **env_overrides):
|
|
return self.run_script(self.SCRIPT, **env_overrides)
|
|
|
|
# -- happy path ------------------------------------------------------
|
|
def test_publish_replaces_live_pem_and_archives_the_previous_one(self):
|
|
self.seed_previous_bundle()
|
|
result = self.run_it()
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertEqual(read(self.live_pem), NEW_BUNDLE,
|
|
'live pem is not the newly assembled cert+key')
|
|
backup = os.path.join(self.backup_dir, DOMAIN + '.pem')
|
|
self.assertTrue(os.path.exists(backup),
|
|
f'previous bundle was not archived to {backup}')
|
|
self.assertEqual(read(backup), PREVIOUS_BUNDLE,
|
|
'the archived bundle is not the one that was replaced')
|
|
self.assertIn('1 updated, 0 failed', self.logs())
|
|
self.assertTrue(self.reload_attempted(),
|
|
'HAProxy was never reloaded after a successful update')
|
|
self.assert_certs_dir_is_clean()
|
|
self.assert_no_staging_leftovers()
|
|
|
|
def test_first_publish_works_with_no_previous_bundle(self):
|
|
result = self.run_it()
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertEqual(read(self.live_pem), NEW_BUNDLE)
|
|
self.assert_certs_dir_is_clean()
|
|
|
|
# -- THE HEADLINE ----------------------------------------------------
|
|
def test_unreadable_source_key_leaves_the_previous_bundle_intact(self):
|
|
"""The bug this whole change exists for.
|
|
|
|
Pre-fix: `cat cert key > live.pem` truncated live.pem before cat ran,
|
|
so an unreadable key left a key-less (or empty) pem in the directory
|
|
HAProxy loads wholesale -> the :443 bind fails -> every site on the
|
|
host loses HTTPS. Checking cat's exit status did not undo that.
|
|
"""
|
|
self.seed_previous_bundle()
|
|
before = read(self.live_pem)
|
|
self.assertTrue(self.bundle_is_valid(self.live_pem),
|
|
'fixture precondition: the seeded bundle must be valid')
|
|
|
|
how = self.break_source_key()
|
|
|
|
result = self.run_it()
|
|
|
|
self.assertEqual(read(self.live_pem), before,
|
|
f'the live pem was damaged by a failed publish ({how}); '
|
|
f'HAProxy would fail to load the certs directory')
|
|
self.assertTrue(self.bundle_is_valid(self.live_pem),
|
|
'the pem left on disk is no longer a usable bundle')
|
|
logs = self.logs()
|
|
self.assertIn(f'Failed to combine certificate for {DOMAIN}', logs)
|
|
self.assertIn('0 updated, 1 failed', logs)
|
|
self.assertFalse(self.reload_attempted(),
|
|
'HAProxy was reloaded even though nothing was updated')
|
|
self.assert_certs_dir_is_clean()
|
|
self.assert_no_staging_leftovers()
|
|
self.assertEqual(result.returncode, 0,
|
|
'a per-domain failure should not change the exit code')
|
|
|
|
def break_source_key(self):
|
|
"""Make reading the source key fail, however this environment allows.
|
|
|
|
chmod 000 is the faithful reproduction (file present, `-f` true, cat
|
|
fails), but it is a no-op for root, so as root we truncate the key
|
|
instead: pre-fix that is even nastier, because `cat` then *succeeds*
|
|
and silently publishes a key-less pem.
|
|
"""
|
|
if os.geteuid() == 0:
|
|
write(self.src_key, '')
|
|
return 'zero-length source key (running as root)'
|
|
os.chmod(self.src_key, 0o000)
|
|
return 'unreadable source key (chmod 000)'
|
|
|
|
# -- other ways to end up with an unusable bundle --------------------
|
|
def test_cert_only_bundle_is_rejected(self):
|
|
self.seed_previous_bundle()
|
|
before = read(self.live_pem)
|
|
write(self.src_key, TEST_CERT) # no private key block at all
|
|
|
|
self.run_it()
|
|
|
|
self.assertEqual(read(self.live_pem), before,
|
|
'a key-less bundle was published over the live pem')
|
|
self.assertIn('0 updated, 1 failed', self.logs())
|
|
self.assertFalse(self.reload_attempted())
|
|
self.assert_certs_dir_is_clean()
|
|
self.assert_no_staging_leftovers()
|
|
|
|
def test_truncated_certificate_block_is_rejected(self):
|
|
self.seed_previous_bundle()
|
|
before = read(self.live_pem)
|
|
write(self.src_cert, TEST_CERT.split('\n')[0] + '\nMIIDFzCCAf+gAwIBA\n')
|
|
|
|
self.run_it()
|
|
|
|
self.assertEqual(read(self.live_pem), before,
|
|
'a truncated certificate was published over the live pem')
|
|
self.assertIn('0 updated, 1 failed', self.logs())
|
|
self.assert_certs_dir_is_clean()
|
|
|
|
def test_mismatched_key_is_rejected(self):
|
|
if shutil.which('openssl') is None:
|
|
self.skipTest('openssl CLI not available: the cert/key pairing '
|
|
'check is best-effort and is skipped by design')
|
|
self.seed_previous_bundle()
|
|
before = read(self.live_pem)
|
|
write(self.src_key, UNRELATED_KEY)
|
|
|
|
self.run_it()
|
|
|
|
self.assertEqual(read(self.live_pem), before,
|
|
'a bundle whose key does not match the cert was published')
|
|
self.assertIn('does not match the certificate', self.logs())
|
|
self.assertIn('0 updated, 1 failed', self.logs())
|
|
self.assert_certs_dir_is_clean()
|
|
|
|
# -- the certs directory is HAProxy's, not ours ----------------------
|
|
def test_no_stray_files_in_certs_dir_after_success_or_failure(self):
|
|
self.seed_previous_bundle()
|
|
self.run_it()
|
|
self.assert_certs_dir_is_clean()
|
|
self.assertEqual(sorted(os.listdir(self.certs_dir)), [DOMAIN + '.pem'])
|
|
|
|
self.break_source_key()
|
|
self.run_it()
|
|
self.assert_certs_dir_is_clean()
|
|
self.assertEqual(sorted(os.listdir(self.certs_dir)), [DOMAIN + '.pem'])
|
|
self.assert_no_staging_leftovers()
|
|
|
|
def test_staging_and_backup_dirs_are_outside_the_certs_dir(self):
|
|
"""Belt and braces: even with the defaults, nothing lands under certs/."""
|
|
self.seed_previous_bundle()
|
|
# Drop the explicit overrides so the derived defaults are exercised.
|
|
env = {'CERT_STAGING_DIR': '', 'CERT_BACKUP_DIR': ''}
|
|
self.run_it(**env)
|
|
|
|
self.assert_certs_dir_is_clean()
|
|
self.assertEqual(sorted(os.listdir(self.certs_dir)), [DOMAIN + '.pem'])
|
|
self.assertTrue(
|
|
os.path.exists(os.path.join(self.haproxy_dir, 'cert-staging')),
|
|
'default staging dir is not the documented sibling of the certs dir')
|
|
self.assertTrue(
|
|
os.path.exists(os.path.join(self.haproxy_dir, 'cert-backups',
|
|
DOMAIN + '.pem')),
|
|
'default backup dir is not the documented sibling of the certs dir')
|
|
|
|
# -- reload gating ---------------------------------------------------
|
|
def test_reload_is_not_attempted_when_haproxy_config_is_invalid(self):
|
|
write(self.haproxy_cfg, GOOD_HAPROXY_CFG + BROKEN_TOKEN + '\n')
|
|
|
|
result = self.run_it()
|
|
|
|
self.assertFalse(self.reload_attempted(),
|
|
'HAProxy was reloaded with a configuration that '
|
|
'`haproxy -c` rejects')
|
|
self.assertNotEqual(result.returncode, 0,
|
|
'refusing to reload must be a loud, non-zero exit')
|
|
self.assertIn('does not validate', self.logs())
|
|
|
|
def test_reload_happens_when_the_config_validates(self):
|
|
self.run_it()
|
|
self.assertTrue(self.reload_attempted())
|
|
self.assertIn('reload', read(self.socat_log))
|
|
|
|
def test_reload_is_not_attempted_when_nothing_was_updated(self):
|
|
shutil.rmtree(self.domain_dir)
|
|
result = self.run_it()
|
|
self.assertEqual(result.returncode, 0)
|
|
self.assertFalse(self.reload_attempted())
|
|
|
|
|
|
class TestRenewCertificates(CertScriptBehaviour, CertScriptFixture):
|
|
SCRIPT = 'renew-certificates.sh'
|
|
|
|
|
|
class TestSyncCertificates(CertScriptBehaviour, CertScriptFixture):
|
|
SCRIPT = 'sync-certificates.sh'
|
|
|
|
|
|
class TestCertPublishLibrary(CertScriptFixture):
|
|
"""Unit-level checks on cert-publish-lib.sh itself."""
|
|
|
|
def call(self, snippet, *args):
|
|
return subprocess.run(
|
|
['bash', '-c', '. "$1"; shift; ' + snippet, '_', LIB, *args],
|
|
env=self.env(), capture_output=True, text=True)
|
|
|
|
def test_valid_bundle_accepted(self):
|
|
path = write(os.path.join(self.tmp, 'ok.pem'), NEW_BUNDLE)
|
|
self.assertEqual(self.call('cert_bundle_valid "$1"', path).returncode, 0)
|
|
|
|
def test_empty_and_missing_bundles_rejected(self):
|
|
empty = write(os.path.join(self.tmp, 'empty.pem'), '')
|
|
self.assertNotEqual(self.call('cert_bundle_valid "$1"', empty).returncode, 0)
|
|
missing = os.path.join(self.tmp, 'nope.pem')
|
|
self.assertNotEqual(self.call('cert_bundle_valid "$1"', missing).returncode, 0)
|
|
|
|
def test_key_without_end_marker_rejected(self):
|
|
truncated = write(os.path.join(self.tmp, 'cut.pem'),
|
|
TEST_CERT + '-----BEGIN PRIVATE KEY-----\nMIIEvAIB\n')
|
|
self.assertNotEqual(self.call('cert_bundle_valid "$1"', truncated).returncode, 0)
|
|
|
|
def test_a_broken_live_pem_does_not_overwrite_a_good_backup(self):
|
|
"""Mirrors create_backup(require_valid=True) in haproxy_manager.py.
|
|
|
|
If the pem currently on disk is already garbage, archiving it would
|
|
replace a restorable backup with an unusable one.
|
|
"""
|
|
os.makedirs(self.backup_dir)
|
|
good_backup = write(os.path.join(self.backup_dir, DOMAIN + '.pem'),
|
|
PREVIOUS_BUNDLE)
|
|
write(self.live_pem, 'garbage, not a pem at all\n')
|
|
|
|
result = self.call('cert_publish "$1" "$2" "$3"',
|
|
self.src_cert, self.src_key, self.live_pem)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertEqual(read(self.live_pem), NEW_BUNDLE)
|
|
self.assertEqual(read(good_backup), PREVIOUS_BUNDLE,
|
|
'a good backup was overwritten with an unusable pem')
|
|
|
|
def test_publish_fails_loudly_when_the_rename_cannot_happen(self):
|
|
"""No silent fallback to writing straight into the certs dir."""
|
|
self.seed_previous_bundle()
|
|
before = read(self.live_pem)
|
|
os.chmod(self.certs_dir, 0o500) # no writes: mv will fail
|
|
self.addCleanup(os.chmod, self.certs_dir, 0o755)
|
|
if os.geteuid() == 0:
|
|
self.skipTest('root ignores directory permissions')
|
|
|
|
result = self.call('cert_publish "$1" "$2" "$3"',
|
|
self.src_cert, self.src_key, self.live_pem)
|
|
|
|
self.assertNotEqual(result.returncode, 0,
|
|
'a failed rename was reported as success')
|
|
self.assertEqual(read(self.live_pem), before,
|
|
'the live pem was damaged by a failed rename')
|
|
self.assert_no_staging_leftovers()
|
|
|
|
def test_haproxy_config_ok_follows_the_validator(self):
|
|
self.assertEqual(self.call('haproxy_config_ok').returncode, 0)
|
|
write(self.haproxy_cfg, GOOD_HAPROXY_CFG + BROKEN_TOKEN + '\n')
|
|
self.assertNotEqual(self.call('haproxy_config_ok').returncode, 0)
|
|
|
|
def test_missing_openssl_warns_but_does_not_block(self):
|
|
"""Best-effort layer: a missing checker must not stall renewals."""
|
|
fake_path = os.path.join(self.tmp, 'no-openssl-bin')
|
|
os.makedirs(fake_path)
|
|
for tool in ('cat', 'grep', 'mktemp', 'mv', 'cp', 'rm', 'mkdir',
|
|
'basename', 'dirname', 'find', 'date', 'chmod'):
|
|
real = shutil.which(tool)
|
|
if real:
|
|
os.symlink(real, os.path.join(fake_path, tool))
|
|
path = write(os.path.join(self.tmp, 'ok.pem'), NEW_BUNDLE)
|
|
|
|
# bash by absolute path: the stripped PATH cannot resolve it.
|
|
result = subprocess.run(
|
|
[shutil.which('bash'), '-c', '. "$1"; cert_bundle_valid "$2"',
|
|
'_', LIB, path],
|
|
env=self.env(PATH=fake_path), capture_output=True, text=True)
|
|
|
|
self.assertEqual(result.returncode, 0,
|
|
'a missing openssl blocked publication')
|
|
self.assertRegex(result.stdout + result.stderr,
|
|
r'(?i)warning.*openssl',
|
|
'the skipped pairing check was not announced loudly')
|
|
|
|
def test_missing_openssl_still_rejects_a_structurally_broken_bundle(self):
|
|
fake_path = os.path.join(self.tmp, 'no-openssl-bin2')
|
|
os.makedirs(fake_path)
|
|
for tool in ('cat', 'grep', 'date'):
|
|
real = shutil.which(tool)
|
|
if real:
|
|
os.symlink(real, os.path.join(fake_path, tool))
|
|
path = write(os.path.join(self.tmp, 'nokey.pem'), TEST_CERT)
|
|
|
|
# bash by absolute path: the stripped PATH cannot resolve it.
|
|
result = subprocess.run(
|
|
[shutil.which('bash'), '-c', '. "$1"; cert_bundle_valid "$2"',
|
|
'_', LIB, path],
|
|
env=self.env(PATH=fake_path), capture_output=True, text=True)
|
|
|
|
self.assertNotEqual(result.returncode, 0,
|
|
'structural checks stopped being mandatory')
|
|
|
|
|
|
class TestScriptsAreSane(unittest.TestCase):
|
|
"""Cheap static guards against the failure mode coming back."""
|
|
|
|
SHELL_FILES = ('renew-certificates.sh', 'sync-certificates.sh',
|
|
'cert-publish-lib.sh')
|
|
|
|
def test_shell_files_parse(self):
|
|
for name in self.SHELL_FILES:
|
|
path = os.path.join(SCRIPTS_DIR, name)
|
|
result = subprocess.run(['bash', '-n', path],
|
|
capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, 0,
|
|
f'{name}: {result.stderr}')
|
|
|
|
def test_no_script_redirects_into_the_live_pem(self):
|
|
pattern = re.compile(r'>\s*"?\$\{?COMBINED_FILE')
|
|
for name in ('renew-certificates.sh', 'sync-certificates.sh'):
|
|
body = read(os.path.join(SCRIPTS_DIR, name))
|
|
self.assertIsNone(pattern.search(body),
|
|
f'{name} still redirects output straight into the '
|
|
f'live pem HAProxy is serving')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
print(f'testing scripts from: {SCRIPTS_DIR}')
|
|
unittest.main(verbosity=2)
|