diff --git a/CLAUDE.md b/CLAUDE.md index e2e69d4..5b4df66 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **Stick-table contract**: `python3 scripts/test-stick-table-contract.py` - offline; holds the templates' `store` clauses, `STICK_TABLE_FIELD_CONTRACT`, and every consumer to each other. Run it after touching any `stick-table` line. - **Runtime-map contract**: `python3 scripts/test-runtime-map-contract.py` - offline; asserts the runtime map commands are `@1`-prefixed, reference the map by FILE PATH (never `#`), carry the value `1`, and that every captured rejection is classified as a failure. Run it after touching any `add map`/`del map`/`clear map` path. - **Certificate destruction safety**: `python3 scripts/test-cert-write-safety.py` - offline; asserts a live `.pem` is never truncated, removed, or its certbot lineage deleted while any configured domain still references it (one bundle serves many names, so `ssl_cert_path` is routinely shared). Run it after touching any `os.remove`/`certbot delete`/PEM-write path. -- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - moved to `cert-retired/`, `live-retired/`, `renewal-retired/`, never deleted - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`. +- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired//` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`. - **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md ### Reading stick tables (and why it is easy to get silently wrong) diff --git a/Dockerfile b/Dockerfile index f66654f..094371b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,8 @@ WORKDIR /haproxy COPY ./templates /haproxy/templates COPY requirements.txt /haproxy/ COPY haproxy_manager.py /haproxy/ +# Read by GET /api/capabilities (WHP gates whole-site re-bundling on it). +COPY VERSION /haproxy/VERSION COPY scripts /haproxy/scripts COPY trusted_ips.list /etc/haproxy/trusted_ips.list COPY trusted_ips.map /etc/haproxy/trusted_ips.map diff --git a/haproxy_manager.py b/haproxy_manager.py index a04d23d..66dee7b 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -774,6 +774,30 @@ def health_check(): 'error': str(e) }), 500 +# What this edge supports. WHP's orchestrator only switches to whole-site +# re-bundling with unreachable_names when it sees these (HARD RULE: the WHP side +# must not rely on 2026.10.3 behaviour against an older image, whose CN-only +# quarantine + `certbot delete` would act on the larger bundles). +EDGE_CAPABILITIES = ['bundle_renewed_flag', 'san_overlap_retirement', + 'unreachable_names', 'retired_lineage_tree'] + + +def _image_version(): + for path in ('/haproxy/VERSION', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'VERSION')): + try: + with open(path) as fh: + return fh.read().strip() + except OSError: + continue + return 'unknown' + + +@app.route('/api/capabilities', methods=['GET']) +@require_api_key +def get_capabilities(): + return jsonify({'version': _image_version(), 'capabilities': EDGE_CAPABILITIES}) + + @app.route('/api/regenerate', methods=['GET']) @require_api_key def regenerate_conf(): @@ -1017,9 +1041,11 @@ def request_ssl(): # would lose its last valid certificate (the shared-PEM hazard: one bundle # routinely serves several sites' names, see domains_referencing_cert_path); # * nothing is deleted. The pem goes to the cert-retired sibling directory and -# the lineage's live/ and renewal/.conf go to live-retired/ and -# renewal-retired/, so certbot stops renewing it and renew-certificates.sh -# stops re-publishing it. Every move is logged and can be reversed with mv. +# the lineage's live/, archive/ and renewal/.conf go +# together to /etc/letsencrypt/retired// (retire_lineage_tree), so +# certbot stops renewing it, renew-certificates.sh stops re-publishing it, +# and the name can still be issued again later. Every move is logged and +# can be reversed with mv. # (This used to `certbot delete` the lineage, which is irreversible.) # Where certbot keeps its lineages. A module global so the tests can repoint @@ -1249,14 +1275,58 @@ def _quarantine_superseded_certs(keep_path, keep_lineage, bundle_names, return summary +def retire_lineage_tree(lineage, stamp=None): + """Move a certbot lineage COMPLETELY out of certbot's view, without deleting. + + live/, archive/ and renewal/.conf go together to + LETSENCRYPT_DIR/retired//{live,archive,renewal}/. All three must + move: certbot's storage.RenewableCert.new_lineage() refuses to create a + lineage while archive/ exists ("archive directory exists for ") - and + it has already written an empty renewal/.conf by then, which breaks every + later --cert-name and every `certbot renew` parse (review H2, verified + against certbot 2.11). Moving only live/ and renewal/ (as the first version + of this did) therefore made the name permanently un-issuable. + + The retired tree keeps the same depth as /etc/letsencrypt, so the relative + live//*.pem -> ../../archive//*N.pem symlinks still resolve, and a + restore is three mv's back. Returns {'retired_to': dir, 'moved': [...]}. + Raises on failure after moving back anything already moved. + """ + stamp = stamp or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + root = os.path.join(LETSENCRYPT_DIR, 'retired', stamp) + n = 1 + while any(os.path.exists(os.path.join(root, part, name)) for part, name in + (('live', lineage), ('archive', lineage), ('renewal', f'{lineage}.conf'))): + root = os.path.join(LETSENCRYPT_DIR, 'retired', f'{stamp}.{n}') + n += 1 + plan = [(os.path.join(LETSENCRYPT_DIR, part, name), os.path.join(root, part, name)) + for part, name in (('renewal', f'{lineage}.conf'), ('live', lineage), + ('archive', lineage))] + moved = [] + try: + for src, dst in plan: + if os.path.lexists(src): + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.move(src, dst) + moved.append((src, dst)) + except Exception: + for src, dst in reversed(moved): + try: + shutil.move(dst, src) + except Exception as e: # pragma: no cover - logged, nothing else to do + logger.critical("Could not restore %s from %s: %s", src, dst, e) + raise + return {'retired_to': root, 'moved': [d for _, d in moved]} + + def _retire_superseded_lineages(keep_lineage, bundle_names, unreachable_names=(), keep_fingerprint=None): - """Move overlapping certbot lineages out of live/ and renewal/. + """Move overlapping certbot lineages out of certbot's tree (retire_lineage_tree). Same overlap and safety rules as _quarantine_superseded_certs(), evaluated against the pem files that are left in the crt dir. Runs only after HAProxy - has loaded the new bundle. Moving live/ to live-retired/ (same depth) - keeps its relative ../../archive symlinks valid, so `mv` back restores it. + has loaded the new bundle. keep_lineage is the bundle's --cert-name and is + never retired. """ summary = {'retired': [], 'kept': [], 'errors': []} live_root = os.path.join(LETSENCRYPT_DIR, 'live') @@ -1275,6 +1345,8 @@ def _retire_superseded_lineages(keep_lineage, bundle_names, unreachable_names=() for lineage in sorted(os.listdir(live_root)): lineage_dir = os.path.join(live_root, lineage) + # Hard guard: the bundle's own --cert-name lineage is never a candidate, + # whatever any other check says (review H1). if lineage == keep_lineage or not os.path.isdir(lineage_dir): continue cert_file = os.path.join(lineage_dir, 'cert.pem') @@ -1299,24 +1371,12 @@ def _retire_superseded_lineages(keep_lineage, bundle_names, unreachable_names=() continue entry = {'lineage': lineage, 'names': sorted(info['names'])} try: - live_dest_root = os.path.join(LETSENCRYPT_DIR, 'live-retired') - os.makedirs(live_dest_root, exist_ok=True) - live_dest = _unique_dest(live_dest_root, lineage) - shutil.move(lineage_dir, live_dest) - entry['live_moved_to'] = live_dest - conf = os.path.join(LETSENCRYPT_DIR, 'renewal', f'{lineage}.conf') - if os.path.exists(conf): - conf_dest_root = os.path.join(LETSENCRYPT_DIR, 'renewal-retired') - os.makedirs(conf_dest_root, exist_ok=True) - conf_dest = _unique_dest(conf_dest_root, f'{lineage}.conf') - shutil.move(conf, conf_dest) - entry['renewal_moved_to'] = conf_dest + entry.update(retire_lineage_tree(lineage)) logger.warning( "Retired certbot lineage %s (names: %s) because it overlaps the " - "new %s bundle: live -> %s, renewal conf -> %s. Archive left in " - "place; reverse with mv to restore.", lineage, - ', '.join(entry['names']), keep_lineage, entry['live_moved_to'], - entry.get('renewal_moved_to', '(none)')) + "new %s bundle: live, archive and renewal conf moved to %s. " + "Restore with three mv's back.", lineage, + ', '.join(entry['names']), keep_lineage, entry['retired_to']) summary['retired'].append(entry) except Exception as e: entry['error'] = str(e) @@ -1419,10 +1479,16 @@ def request_ssl_bundle(): # Defensive: clear any stale lock left by a SIGKILLed prior run. clear_stale_certbot_locks() - # fullchain.pem starts with the leaf, so it identifies the certificate. - before_dir = find_certbot_live_dir(primary) - before_info = read_cert_info(os.path.join(before_dir, 'fullchain.pem')) \ - if before_dir else None + # The lineage is EXACTLY live/: certbot was given + # --cert-name , which is authoritative. find_certbot_live_dir() + # must NOT be used here - it returns the newest of and + # -NNNN, so after a "not yet due" no-op a stale, more recently + # renewed -0001 would be published as .pem and the + # real lineage would then be retired as "superseded" (review + # H1, reproduced with brainjar.net). fullchain.pem starts with the leaf. + lineage_dir = os.path.join(LETSENCRYPT_DIR, 'live', primary) + before_info = read_cert_info(os.path.join(lineage_dir, 'fullchain.pem')) \ + if os.path.isdir(lineage_dir) else None result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: @@ -1436,9 +1502,8 @@ def request_ssl_bundle(): 'attempted_names': names, }), 500 - # Locate the lineage. With --cert-name primary, this should be a - # stable directory name (no -NNNN suffix on the first issuance). - live_dir = find_certbot_live_dir(primary) + # --cert-name means the lineage is live/, exactly. + live_dir = lineage_dir if os.path.isdir(lineage_dir) else None if not live_dir: error_msg = f'Bundle issued but live dir not found for {primary}' log_operation('request_ssl_bundle', False, error_msg) @@ -1501,7 +1566,7 @@ def request_ssl_bundle(): # only copy we have. cleanup_summary = _quarantine_superseded_certs( keep_path=combined_path, - keep_lineage=os.path.basename(live_dir), + keep_lineage=primary, bundle_names=set(names), unreachable_names=unreachable_names, ) @@ -1525,7 +1590,7 @@ def request_ssl_bundle(): # Only now that HAProxy has loaded the new bundle: stop certbot renewing # (and renew-certificates.sh re-publishing) overlapping old lineages. lineage_summary = _retire_superseded_lineages( - keep_lineage=os.path.basename(live_dir), + keep_lineage=primary, bundle_names=set(names), unreachable_names=unreachable_names, keep_fingerprint=new_info.get('fingerprint') if new_info else None, @@ -1546,6 +1611,7 @@ def request_ssl_bundle(): 'names': names, 'cert_path': combined_path, 'renewed': bool(renewed), + 'capabilities': EDGE_CAPABILITIES, 'certbot_not_due': bool(certbot_not_due), 'not_after': not_after, 'not_after_iso': datetime.fromtimestamp(not_after, timezone.utc).isoformat() diff --git a/scripts/cert-publish-lib.sh b/scripts/cert-publish-lib.sh index fcf2036..f8e0a9e 100644 --- a/scripts/cert-publish-lib.sh +++ b/scripts/cert-publish-lib.sh @@ -228,6 +228,57 @@ cert_not_after() { openssl x509 -enddate -noout -in "$1" 2>/dev/null /{cert,chain,fullchain,privkey}1.pem + live//*.pem + symlinks (../../archive//...1.pem) + renewal/.conf, like certbot.""" + archive = os.path.join(le_dir, 'archive', lineage) + live = os.path.join(le_dir, 'live', lineage) + os.makedirs(archive, exist_ok=True) + os.makedirs(live, exist_ok=True) + for kind, body in (('cert', cert), ('chain', cert), ('fullchain', cert), ('privkey', key)): + with open(os.path.join(archive, f'{kind}1.pem'), 'w') as fh: + fh.write(body) + link = os.path.join(live, f'{kind}.pem') + if os.path.lexists(link): + os.remove(link) + os.symlink(f'../../archive/{lineage}/{kind}1.pem', link) + os.makedirs(os.path.join(le_dir, 'renewal'), exist_ok=True) + with open(os.path.join(le_dir, 'renewal', f'{lineage}.conf'), 'w') as fh: + fh.write(f'archive_dir = {archive}\ncert = {live}/cert.pem\n') + return live + + +def retired_path(le_dir, part, name): + hits = sorted(__import__('glob').glob(os.path.join(le_dir, 'retired', '*', part, name))) + return hits[-1] if hits else os.path.join(le_dir, 'retired', '', part, name) + + FIX_ONLY_RETIRE = unittest.skipIf( TESTING_FOREIGN_TREE and not hasattr(hm, 'retirement_blockers'), 'HAPROXY_MANAGER_DIR points at a tree without SAN-overlap retirement') @@ -738,13 +764,14 @@ class TestSanOverlapRetirement(CertPublishTestCase): for name in ('brainjar.net', 'www.brainjar.net'): self.add_domain(name, name.replace('.', '_')) - def old_lineage(self, lineage, fixture): - live = os.path.join(self.le_live, lineage) - self.write(os.path.join(live, 'fullchain.pem'), san(f'{fixture}.crt')) - self.write(os.path.join(live, 'cert.pem'), san(f'{fixture}.crt')) - self.write(os.path.join(live, 'privkey.pem'), san(f'{fixture}.key')) - conf = self.write(os.path.join(self.le, 'renewal', f'{lineage}.conf'), - f'# renewal conf for {lineage}\n') + def old_lineage(self, lineage, fixture, publish=True): + """A lineage laid out the way certbot does it: real files in + archive//*1.pem, relative symlinks in live//, a renewal conf.""" + live = make_certbot_lineage(self.le, lineage, san(f'{fixture}.crt'), + san(f'{fixture}.key')) + conf = os.path.join(self.le, 'renewal', f'{lineage}.conf') + if not publish: + return live, conf, None pem = self.write(os.path.join(self.certs, f'{lineage}.pem'), san(f'{fixture}.crt') + san(f'{fixture}.key')) return live, conf, pem @@ -775,8 +802,14 @@ class TestSanOverlapRetirement(CertPublishTestCase): # re-publish it and certbot can no longer try to renew it. self.assertFalse(os.path.exists(live)) self.assertFalse(os.path.exists(conf)) - self.assertTrue(os.path.isdir(os.path.join(self.le, 'live-retired', 'brain-jar.com'))) - self.assertTrue(os.path.exists(os.path.join(self.le, 'renewal-retired', 'brain-jar.com.conf'))) + # live, archive AND renewal leave together (review H2), into one tree. + self.assertFalse(os.path.exists(os.path.join(self.le, 'archive', 'brain-jar.com'))) + self.assertTrue(os.path.isdir(retired_path(self.le, 'live', 'brain-jar.com'))) + self.assertTrue(os.path.isdir(retired_path(self.le, 'archive', 'brain-jar.com'))) + self.assertTrue(os.path.exists(retired_path(self.le, 'renewal', 'brain-jar.com.conf'))) + # Relative symlinks still resolve in the retired tree (restore = mv back). + self.assertIn('BEGIN CERTIFICATE', self.read( + os.path.join(retired_path(self.le, 'live', 'brain-jar.com'), 'cert.pem'))) self.assertEqual([], self.certbot_deletes()) self.assertTrue(os.path.exists(os.path.join(self.certs, 'brainjar.net.pem'))) self.assertTrue(self.edge_would_start()) @@ -857,6 +890,95 @@ class TestSanOverlapRetirement(CertPublishTestCase): self.assertIn('--keep-until-expiring', self.certbot_invocations()[-1]) + + @FIX_ONLY_RETIRE + @NEEDS_OPENSSL + def test_not_due_noop_never_retires_its_own_cert_name_lineage(self): + """Review H1: find_certbot_live_dir() returns the NEWEST of brainjar.net + and brainjar.net-0001. After a certbot no-op that was the stale -0001: + it got published as brainjar.net.pem and the real brainjar.net lineage + was retired as "superseded".""" + del os.environ['TEST_CERTBOT_INSTALL_SRC'] + os.environ['TEST_CERTBOT_NOT_DUE'] = '1' + real = make_certbot_lineage(self.le, 'brainjar.net', san('new_bundle.crt'), san('new_bundle.key')) + stale = make_certbot_lineage(self.le, 'brainjar.net-0001', san('dead_old.crt'), san('dead_old.key')) + later = time.time() + 60 + os.utime(os.path.join(self.le, 'archive', 'brainjar.net-0001', 'fullchain1.pem'), (later, later)) + # Production behaviour of find_certbot_live_dir(): newest wins. + hm.find_certbot_live_dir = lambda base: stale if base == 'brainjar.net' else None + + data = self.bundle() + + self.assertTrue(os.path.isdir(real), 'the --cert-name lineage itself was retired') + self.assertTrue(os.path.exists(os.path.join(self.le, 'renewal', 'brainjar.net.conf'))) + self.assertNotIn('brainjar.net', [e['lineage'] for e in data['cleanup']['lineages']['retired']]) + published = self.read(os.path.join(self.certs, 'brainjar.net.pem')) + self.assertIn(san('new_bundle.crt').strip().splitlines()[1], published, + 'published the stale -0001 certificate instead of live/brainjar.net') + self.assertFalse(data['renewed']) + + @FIX_ONLY_RETIRE + def test_bundle_reports_capabilities(self): + data = self.bundle() + self.assertIn('san_overlap_retirement', data.get('capabilities', [])) + caps = self.client.get('/api/capabilities').get_json() + self.assertIn('unreachable_names', caps['capabilities']) + + +class TestRetiredLineageCanBeReissued(CertPublishTestCase): + """Review H2: retirement must leave certbot able to issue the name again.""" + + def setUp(self): + super().setUp() + self.le = hm.LETSENCRYPT_DIR + + @FIX_ONLY_RETIRE + def test_retirement_removes_every_part_certbot_checks(self): + make_certbot_lineage(self.le, 'brain-jar.com', san('expired_old.crt'), san('expired_old.key')) + out = hm.retire_lineage_tree('brain-jar.com') + # certbot new_lineage() refuses if any of these exist for the name. + for part in (('live', 'brain-jar.com'), ('archive', 'brain-jar.com'), + ('renewal', 'brain-jar.com.conf')): + self.assertFalse(os.path.lexists(os.path.join(self.le, *part)), part) + self.assertTrue(os.path.lexists(os.path.join(out['retired_to'], *part)), part) + self.assertIn('BEGIN CERTIFICATE', + self.read(os.path.join(out['retired_to'], 'live', 'brain-jar.com', 'fullchain.pem'))) + + @FIX_ONLY_RETIRE + def test_second_retirement_of_the_same_name_does_not_collide(self): + make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key')) + a = hm.retire_lineage_tree('x.example', stamp='S') + make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key')) + b = hm.retire_lineage_tree('x.example', stamp='S') + self.assertNotEqual(a['retired_to'], b['retired_to']) + self.assertTrue(os.path.isdir(os.path.join(a['retired_to'], 'archive', 'x.example'))) + + @FIX_ONLY_RETIRE + def test_certbot_can_issue_the_name_again_after_retirement(self): + """Against certbot's own storage code (skipped where certbot is not + importable, e.g. the image's app python).""" + try: + from certbot._internal import storage + except ImportError: + self.skipTest('certbot not importable from this python') + import types + cfg = types.SimpleNamespace( + config_dir=self.le, renewal_configs_dir=os.path.join(self.le, 'renewal'), + default_archive_dir=os.path.join(self.le, 'archive'), + live_dir=os.path.join(self.le, 'live')) + real_rv = storage.relevant_values + storage.relevant_values = lambda c: {'server': 'https://acme.invalid/directory'} + self.addCleanup(setattr, storage, 'relevant_values', real_rv) + cert, key = san('new_bundle.crt').encode(), san('new_bundle.key').encode() + + first = storage.RenewableCert.new_lineage('brain-jar.com', cert, key, cert, cfg) + self.assertEqual('brain-jar.com', first.lineagename) + hm.retire_lineage_tree('brain-jar.com') + again = storage.RenewableCert.new_lineage('brain-jar.com', cert, key, cert, cfg) + self.assertEqual('brain-jar.com', again.lineagename, + 'certbot could not re-create the lineage under the same name') + self.assertTrue(os.path.getsize(os.path.join(self.le, 'renewal', 'brain-jar.com.conf')) > 0) + class TestExpiredCertificateIsNeverPublished(CertPublishTestCase): @FIX_ONLY_RETIRE