Merge fix/ssl-renewal-durable into release/2026.10.3
Build and push coraza-spoa / Build-and-Push (push) Successful in 3m9s
HAProxy Manager Build and Push / Build-and-Push (push) Failing after 5m7s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-10-09 15:22:52 -07:00
commit 765b2ce00a
18 files changed
+1315 -114

No files matched your search

+81
View File
@@ -203,6 +203,82 @@ cert_bundle_valid() {
return 0
}
# cert_is_expired CERT_FILE
#
# Returns 0 only when openssl positively reports that the first certificate in
# CERT_FILE has expired. Anything else (not expired, unreadable, no openssl)
# returns 1; unreadable material is rejected with a better message by
# cert_bundle_valid().
#
# Why: HAProxy loads the crt directory wholesale and serves the FIRST file that
# claims an SNI name. An expired bundle in there is never useful and can only
# shadow a valid one - on 2026-10-07 renew-certificates.sh kept re-publishing
# the expired brain-jar.com lineage every 12 hours, and HAProxy served it for
# brainjar.net instead of the valid brainjar.net bundle next to it.
cert_is_expired() {
local file="$1" out
[ -f "$file" ] || return 1
command -v openssl >/dev/null 2>&1 || return 1
out="$(openssl x509 -checkend 0 -noout -in "$file" 2>/dev/null </dev/null)"
[ $? -eq 1 ] && [[ "$out" == *"will expire"* ]]
}
# cert_not_after CERT_FILE - the notAfter date string, or empty.
cert_not_after() {
openssl x509 -enddate -noout -in "$1" 2>/dev/null </dev/null | sed 's/^notAfter=//'
}
cert_retired_dir() {
if [ -n "${CERT_RETIRED_DIR:-}" ]; then
echo "$CERT_RETIRED_DIR"
else
echo "$(dirname "${SSL_CERTS_DIR:-/etc/haproxy/certs}")/cert-retired"
fi
}
# cert_retire_expired_pems
#
# Move every EXPIRED *.pem out of $SSL_CERTS_DIR into cert-retired/ (a sibling,
# never inside the crt dir). Sets CERT_RETIRED_EXPIRED to the number moved.
#
# Not publishing expired lineages stops the re-publish loop, but a pem that was
# published while valid and has expired since stays in the crt dir and keeps
# shadowing - that was exactly brain-jar.com.pem (review M2). An expired
# certificate is never a valid certificate for anyone, so it can always go: the
# only-valid-cert safety rule cannot be violated by moving it. Two exceptions:
# * default_self_signed_cert.pem - HAProxy's fallback, regenerated separately;
# * the LAST .pem in the directory - a crt directory with no certificate at
# all fails the whole :443 bind, which is worse than serving an expired one.
# Nothing is deleted; reverse with mv.
cert_retire_expired_pems() {
local dir="${SSL_CERTS_DIR:-/etc/haproxy/certs}" retired f base dest remaining
CERT_RETIRED_EXPIRED=0
retired="$(cert_retired_dir)"
for f in "$dir"/*.pem; do
[ -f "$f" ] || continue
base="$(basename "$f")"
[ "$base" = "default_self_signed_cert.pem" ] && continue
cert_is_expired "$f" || continue
remaining="$(find "$dir" -maxdepth 1 -type f -name '*.pem' | wc -l)"
if [ "$remaining" -le 1 ]; then
log_error "Expired certificate $f is the only certificate left in $dir - NOT retiring it (an empty crt directory fails the :443 bind)"
continue
fi
if ! mkdir -p "$retired"; then
log_error "cannot create $retired - expired $f left in place"
continue
fi
dest="$retired/$base"
[ -e "$dest" ] && dest="$dest.$(date -u +%Y%m%dT%H%M%SZ)"
if mv -- "$f" "$dest"; then
log_error "Retired EXPIRED certificate $base (notAfter $(cert_not_after "$dest")) from $dir to $dest - it could only shadow valid certificates"
CERT_RETIRED_EXPIRED=$((CERT_RETIRED_EXPIRED + 1))
else
log_error "failed to move expired $f to $dest"
fi
done
}
# cert_publish CERT_FILE KEY_FILE DEST_FILE
#
# Assemble CERT_FILE + KEY_FILE into DEST_FILE without ever exposing a
@@ -227,6 +303,11 @@ cert_publish() {
log_error "cert_publish: private key $key_file is missing or empty"
return 1
fi
# (a2) never publish an expired certificate (see cert_is_expired).
if cert_is_expired "$cert_file"; then
log_error "cert_publish: certificate $cert_file expired ($(cert_not_after "$cert_file")) - refusing to publish $dest_file"
return 1
fi
# (b) assemble in the staging dir - NOT in the certs dir, which HAProxy
# scans wholesale.
+17 -3
View File
@@ -65,13 +65,22 @@ fi
# Copy certificates for each domain
UPDATED=0
FAILED=0
SKIPPED_EXPIRED=0
while read -r domain; do
CERT_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/fullchain.pem"
KEY_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/privkey.pem"
COMBINED_FILE="${SSL_CERTS_DIR}/${domain}.pem"
if [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
if [ -f "$CERT_FILE" ] && cert_is_expired "$CERT_FILE"; then
# Never publish an expired lineage. It cannot help any client and,
# because HAProxy serves the first file claiming an SNI name, it can
# shadow a valid bundle for a name it shares (brainjar.net, 2026-10-07).
# Counted separately from FAILED: a dead lineage that keeps expiring is
# reported on every run (ERROR log) without making every run "fail".
log_error "Skipping EXPIRED certificate lineage $domain (notAfter $(cert_not_after "$CERT_FILE")) - not publishing it"
SKIPPED_EXPIRED=$((SKIPPED_EXPIRED + 1))
elif [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
# Assemble in a staging dir and rename into place. NEVER redirect into
# $COMBINED_FILE: the shell truncates the live pem before cat runs, and
# HAProxy loads $SSL_CERTS_DIR as a directory, so one bad file there
@@ -89,10 +98,15 @@ while read -r domain; do
fi
done <<< "$DOMAINS"
log_info "Certificate update completed: $UPDATED updated, $FAILED failed"
# Already-published pems that have expired since are moved out too (they
# can only shadow; see cert_retire_expired_pems).
cert_retire_expired_pems
RETIRED_EXPIRED=$CERT_RETIRED_EXPIRED
log_info "Certificate update completed: $UPDATED updated, $FAILED failed, $SKIPPED_EXPIRED expired lineage(s) skipped, $RETIRED_EXPIRED expired pem(s) retired"
# Reload HAProxy if any certificates were updated
if [ $UPDATED -gt 0 ]; then
if [ $UPDATED -gt 0 ] || [ "$RETIRED_EXPIRED" -gt 0 ]; then
# Never reload onto unvalidated material: a reload that fails to load the
# certs directory drops HTTPS for every site on this host.
if ! haproxy_config_ok; then
+17 -3
View File
@@ -47,13 +47,22 @@ fi
# Copy certificates for each domain
UPDATED=0
FAILED=0
SKIPPED_EXPIRED=0
while read -r domain; do
CERT_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/fullchain.pem"
KEY_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/privkey.pem"
COMBINED_FILE="${SSL_CERTS_DIR}/${domain}.pem"
if [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
if [ -f "$CERT_FILE" ] && cert_is_expired "$CERT_FILE"; then
# Never publish an expired lineage. It cannot help any client and,
# because HAProxy serves the first file claiming an SNI name, it can
# shadow a valid bundle for a name it shares (brainjar.net, 2026-10-07).
# Counted separately from FAILED: a dead lineage that keeps expiring is
# reported on every run (ERROR log) without making every run "fail".
log_error "Skipping EXPIRED certificate lineage $domain (notAfter $(cert_not_after "$CERT_FILE")) - not publishing it"
SKIPPED_EXPIRED=$((SKIPPED_EXPIRED + 1))
elif [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
# Assemble in a staging dir and rename into place. NEVER redirect into
# $COMBINED_FILE: the shell truncates the live pem before cat runs, and
# HAProxy loads $SSL_CERTS_DIR as a directory, so one bad file there
@@ -71,10 +80,15 @@ while read -r domain; do
fi
done <<< "$DOMAINS"
log_info "Certificate sync completed: $UPDATED updated, $FAILED failed"
# Already-published pems that have expired since are moved out too (they
# can only shadow; see cert_retire_expired_pems).
cert_retire_expired_pems
RETIRED_EXPIRED=$CERT_RETIRED_EXPIRED
log_info "Certificate sync completed: $UPDATED updated, $FAILED failed, $SKIPPED_EXPIRED expired lineage(s) skipped, $RETIRED_EXPIRED expired pem(s) retired"
# Reload HAProxy if any certificates were updated
if [ $UPDATED -gt 0 ]; then
if [ $UPDATED -gt 0 ] || [ "$RETIRED_EXPIRED" -gt 0 ]; then
# Never reload onto unvalidated material: a reload that fails to load the
# certs directory drops HTTPS for every site on this host.
if ! haproxy_config_ok; then
+121
View File
@@ -73,6 +73,15 @@ LIB = os.path.join(SCRIPTS_DIR, 'cert-publish-lib.sh')
BROKEN_TOKEN = '__BROKEN__'
DOMAIN = 'test.example.com'
# Real SAN certificates (see scripts/test-fixtures/san-certs/). expired_old is
# the brain-jar.com shape: a multi-SAN certificate that expired in 2020.
SAN_FIXTURES = os.path.join(SCRIPTS_DIR, 'test-fixtures', 'san-certs')
def san_fixture(name):
with open(os.path.join(SAN_FIXTURES, name)) as fh:
return fh.read()
# --- test key material -------------------------------------------------------
# openssl req -x509 -newkey rsa:2048 -keyout key1 -out cert1 -days 3650 -nodes \
# -subj /CN=test.example.com
@@ -618,6 +627,72 @@ class CertScriptBehaviour:
self.assertFalse(self.reload_attempted())
# -- never publish an expired certificate (brainjar.net, 2026-10-07) ----
def test_expired_lineage_is_skipped_and_reported(self):
"""renew-certificates.sh used to re-publish EVERY live/ lineage every 12h,
expired or not; the expired brain-jar.com bundle then shadowed the valid
brainjar.net bundle for the names they shared."""
dead = os.path.join(self.le_live, 'brain-jar.com')
os.makedirs(dead)
write(os.path.join(dead, 'fullchain.pem'), san_fixture('expired_old.crt'))
write(os.path.join(dead, 'privkey.pem'), san_fixture('expired_old.key'))
result = self.run_it()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertFalse(
os.path.exists(os.path.join(self.certs_dir, 'brain-jar.com.pem')),
'an expired lineage was published into the crt directory')
# The healthy lineage next to it is still published.
self.assertEqual(read(self.live_pem), NEW_BUNDLE)
self.assertIn('Skipping EXPIRED certificate lineage brain-jar.com', self.logs())
self.assertIn('1 updated, 0 failed, 1 expired lineage(s) skipped', self.logs())
self.assertIn('brain-jar.com', read(self.error_log),
'the skip must land in the monitored error log')
def test_expired_lineage_does_not_replace_an_existing_pem(self):
dead = os.path.join(self.le_live, 'brain-jar.com')
os.makedirs(dead)
write(os.path.join(dead, 'fullchain.pem'), san_fixture('expired_old.crt'))
write(os.path.join(dead, 'privkey.pem'), san_fixture('expired_old.key'))
existing = write(os.path.join(self.certs_dir, 'brain-jar.com.pem'), OTHER_BUNDLE)
self.run_it()
self.assertEqual(read(existing), OTHER_BUNDLE,
'an expired lineage overwrote the file in the crt dir')
def test_already_published_expired_pem_is_retired(self):
"""Review M2: skipping publication is not enough - an expired pem that
is ALREADY in the crt dir (brain-jar.com.pem) keeps shadowing."""
expired = write(os.path.join(self.certs_dir, 'brain-jar.com.pem'),
san_fixture('expired_old.crt') + san_fixture('expired_old.key'))
result = self.run_it()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertFalse(os.path.exists(expired), 'expired pem still in the crt dir')
retired = os.path.join(self.haproxy_dir, 'cert-retired', 'brain-jar.com.pem')
self.assertTrue(os.path.exists(retired), 'expired pem was not moved to cert-retired/ (never delete)')
self.assertIn('Retired EXPIRED certificate brain-jar.com.pem', self.logs())
self.assertIn('1 expired pem(s) retired', self.logs())
self.assertTrue(self.reload_attempted())
self.assertEqual(read(self.live_pem), NEW_BUNDLE, 'valid neighbours untouched')
self.assert_certs_dir_is_clean()
def test_retiring_an_expired_pem_alone_triggers_a_reload(self):
self.seed_previous_bundle(NEW_BUNDLE)
shutil.rmtree(self.domain_dir)
other = os.path.join(self.le_live, 'other.example')
os.makedirs(other)
write(os.path.join(other, 'fullchain.pem'), san_fixture('expired_old.crt'))
write(os.path.join(other, 'privkey.pem'), san_fixture('expired_old.key'))
write(os.path.join(self.certs_dir, 'other.example.pem'),
san_fixture('expired_old.crt') + san_fixture('expired_old.key'))
result = self.run_it()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('0 updated', self.logs())
self.assertTrue(self.reload_attempted(), 'HAProxy must reload to drop the retired pem')
class TestRenewCertificates(CertScriptBehaviour, CertScriptFixture):
SCRIPT = 'renew-certificates.sh'
@@ -638,6 +713,52 @@ class TestCertPublishLibrary(CertScriptFixture):
path = write(os.path.join(self.tmp, 'ok.pem'), NEW_BUNDLE)
self.assertEqual(self.call('cert_bundle_valid "$1"', path).returncode, 0)
def test_cert_is_expired_reports_both_directions(self):
expired = write(os.path.join(self.tmp, 'expired.crt'), san_fixture('expired_old.crt'))
valid = write(os.path.join(self.tmp, 'valid.crt'), san_fixture('new_bundle.crt'))
self.assertEqual(0, self.call('cert_is_expired "$1"', expired).returncode,
'an expired certificate was not recognised as expired')
self.assertEqual(1, self.call('cert_is_expired "$1"', valid).returncode,
'a valid certificate was reported as expired')
self.assertEqual(1, self.call('cert_is_expired "$1"',
os.path.join(self.tmp, 'missing')).returncode)
def test_expired_sweep_keeps_the_last_pem_and_the_default_cert(self):
exp = san_fixture('expired_old.crt') + san_fixture('expired_old.key')
only = write(os.path.join(self.certs_dir, 'only.pem'), exp)
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
self.assertIn('n=0', r.stdout)
self.assertTrue(os.path.exists(only), 'emptied the crt dir: that fails the whole :443 bind')
dflt = write(os.path.join(self.certs_dir, 'default_self_signed_cert.pem'), exp)
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
self.assertIn('n=1', r.stdout)
self.assertTrue(os.path.exists(dflt), 'the default self-signed cert is not ours to retire')
self.assertFalse(os.path.exists(only))
def test_expired_sweep_never_touches_a_valid_pem(self):
valid = write(os.path.join(self.certs_dir, 'a.pem'), NEW_BUNDLE)
write(os.path.join(self.certs_dir, 'b.pem'), OTHER_BUNDLE)
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
self.assertIn('n=0', r.stdout)
self.assertEqual(read(valid), NEW_BUNDLE)
def test_cert_publish_refuses_an_expired_certificate(self):
cert = write(os.path.join(self.tmp, 'expired.crt'), san_fixture('expired_old.crt'))
key = write(os.path.join(self.tmp, 'expired.key'), san_fixture('expired_old.key'))
dest = os.path.join(self.certs_dir, 'brain-jar.com.pem')
result = self.call('cert_publish "$1" "$2" "$3"', cert, key, dest)
self.assert_rejected(result, because='expired')
self.assertFalse(os.path.exists(dest), 'expired certificate was published')
# And the same material with a valid certificate IS publishable, so the
# refusal above is about expiry, not about the fixture.
good_cert = write(os.path.join(self.tmp, 'good.crt'), san_fixture('new_bundle.crt'))
good_key = write(os.path.join(self.tmp, 'good.key'), san_fixture('new_bundle.key'))
ok = self.call('cert_publish "$1" "$2" "$3"', good_cert, good_key, dest)
self.assertEqual(0, ok.returncode, ok.stdout + ok.stderr)
self.assertTrue(os.path.exists(dest))
def test_empty_and_missing_bundles_rejected(self):
empty = write(os.path.join(self.tmp, 'empty.pem'), '')
self.assert_rejected(self.call('cert_bundle_valid "$1"', empty),
+389 -6
View File
@@ -57,6 +57,7 @@ import subprocess
import sys
import tempfile
import textwrap
import time
import unittest
BROKEN_TOKEN = '__BROKEN__'
@@ -254,6 +255,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
# certbot stub: succeeds, announces a renewal, and records every invocation so
# tests can assert that `certbot delete` did or did not run.
# TEST_CERTBOT_INSTALL_SRC/_DIR: on certonly, copy fullchain.pem + privkey.pem
# from SRC into DIR (a real issuance changes the lineage's certificate).
# TEST_CERTBOT_NOT_DUE: on certonly, print certbot's "not yet due" wording.
FAKE_CERTBOT = textwrap.dedent("""\
#!/bin/sh
if [ -n "$TEST_CERTBOT_LOG" ]; then
@@ -262,6 +266,14 @@ FAKE_CERTBOT = textwrap.dedent("""\
case "$1" in
renew) echo "Congratulations, all renewals succeeded" ;;
delete) [ -n "$TEST_CERTBOT_DELETE_FAILS" ] && exit 1 ;;
certonly)
if [ -n "$TEST_CERTBOT_INSTALL_SRC" ]; then
mkdir -p "$TEST_CERTBOT_INSTALL_DIR"
cp "$TEST_CERTBOT_INSTALL_SRC/fullchain.pem" "$TEST_CERTBOT_INSTALL_SRC/privkey.pem" "$TEST_CERTBOT_INSTALL_DIR/"
fi
if [ -n "$TEST_CERTBOT_NOT_DUE" ]; then
echo "Certificate not yet due for renewal; no action taken."
fi ;;
esac
exit 0
""")
@@ -331,6 +343,7 @@ class CertPublishTestCase(unittest.TestCase):
'CORAZA_SPOE_BACKUP_PATH': os.path.join(self.etc, 'coraza-spoe.cfg.backup'),
'CLUSTER_SECRET_PATH': os.path.join(self.etc, 'cluster-secret'),
'SSL_CERTS_DIR': self.certs,
'LETSENCRYPT_DIR': os.path.join(self.tmp, 'letsencrypt'),
'HAPROXY_SOCKET_PATH': os.path.join(self.etc, 'haproxy.sock'),
'API_KEY': None,
}
@@ -574,7 +587,7 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
@FIX_ONLY
@NEEDS_OPENSSL
def test_lineage_is_deleted_only_after_haproxy_loads_the_bundle(self):
def test_lineage_is_untouched_when_haproxy_does_not_load_the_bundle(self):
"""A reload failure must leave the lineage intact and the file recoverable."""
old_path = self._setup_supersede(broken=False)
# Make generate_config() produce a config the validator rejects, so the
@@ -591,7 +604,7 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
[], self.certbot_deletes(),
'the lineage must not be deleted when HAProxy did not reload')
self.assertFalse(os.path.exists(old_path))
quarantined = os.path.join(hm.cert_backup_dir(),
quarantined = os.path.join(hm.cert_retired_dir(),
os.path.basename(old_path))
self.assertTrue(os.path.exists(quarantined),
'the superseded file must be recoverable by hand')
@@ -599,8 +612,9 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
@FIX_ONLY
@NEEDS_OPENSSL
def test_successful_bundle_still_supersedes_and_deletes(self):
"""The cleanup must still do its job on the happy path."""
def test_successful_bundle_still_supersedes_without_deleting(self):
"""The cleanup must still do its job on the happy path - by moving, never
by `certbot delete` (irreversible; recovery is a rate-limited ACME order)."""
old_path = self._setup_supersede(broken=False)
resp = self.client.post('/api/ssl/bundle', json={
@@ -612,12 +626,381 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
self.assertFalse(os.path.exists(old_path),
'the superseded file must leave the crt directory or '
'it keeps shadowing the new bundle')
self.assertEqual(['delete --cert-name www.test.example.com -n'],
self.certbot_deletes())
self.assertTrue(os.path.exists(os.path.join(
hm.cert_retired_dir(), os.path.basename(old_path))),
'the superseded file must be moved aside, not destroyed')
self.assertEqual([], self.certbot_deletes(),
'retirement must never `certbot delete` anything')
self.assertTrue(self.edge_would_start())
self.assert_only_final_pems_in_certs_dir()
# --- SAN-overlap retirement (brainjar.net, 2026-10-07) -----------------------
SAN_FIXTURES = os.path.join(MODULE_DIR, 'scripts', 'test-fixtures', 'san-certs')
def san(name):
with open(os.path.join(SAN_FIXTURES, name)) as fh:
return fh.read()
def make_certbot_lineage(le_dir, lineage, cert, key):
"""archive/<x>/{cert,chain,fullchain,privkey}1.pem + live/<x>/*.pem
symlinks (../../archive/<x>/...1.pem) + renewal/<x>.conf, like certbot."""
archive = os.path.join(le_dir, 'archive', lineage)
live = os.path.join(le_dir, 'live', lineage)
os.makedirs(archive, exist_ok=True)
os.makedirs(live, exist_ok=True)
for kind, body in (('cert', cert), ('chain', cert), ('fullchain', cert), ('privkey', key)):
with open(os.path.join(archive, f'{kind}1.pem'), 'w') as fh:
fh.write(body)
link = os.path.join(live, f'{kind}.pem')
if os.path.lexists(link):
os.remove(link)
os.symlink(f'../../archive/{lineage}/{kind}1.pem', link)
os.makedirs(os.path.join(le_dir, 'renewal'), exist_ok=True)
with open(os.path.join(le_dir, 'renewal', f'{lineage}.conf'), 'w') as fh:
fh.write(f'archive_dir = {archive}\ncert = {live}/cert.pem\n')
return live
def retired_path(le_dir, part, name):
hits = sorted(__import__('glob').glob(os.path.join(le_dir, 'retired', '*', part, name)))
return hits[-1] if hits else os.path.join(le_dir, 'retired', '<none>', part, name)
FIX_ONLY_RETIRE = unittest.skipIf(
TESTING_FOREIGN_TREE and not hasattr(hm, 'retirement_blockers'),
'HAPROXY_MANAGER_DIR points at a tree without SAN-overlap retirement')
class TestRetirementRules(unittest.TestCase):
"""retirement_blockers() is the shared-PEM safety rule, tested directly."""
NOW = 2_000_000_000
def cert(self, names, days_left, fp):
return {'names': set(names), 'fingerprint': fp,
'not_after': self.NOW + days_left * 86400, 'cn': sorted(names)[0]}
@FIX_ONLY_RETIRE
def test_expired_candidate_never_blocks(self):
old = self.cert(['brain-jar.com', 'brainjar.net'], -2, 'A')
self.assertEqual([], hm.retirement_blockers(
old, [], {'brainjar.net'}, {'brain-jar.com', 'brainjar.net'}, [], now=self.NOW))
@FIX_ONLY_RETIRE
def test_valid_candidate_that_is_last_cert_for_a_served_name_is_kept(self):
old = self.cert(['shared.example', 'brainjar.net', 'other.example'], 40, 'A')
self.assertEqual(['other.example'], hm.retirement_blockers(
old, [], {'brainjar.net'}, {'brainjar.net', 'other.example'}, [], now=self.NOW))
@FIX_ONLY_RETIRE
def test_unreachable_names_do_not_block(self):
old = self.cert(['dead.example', 'brainjar.net'], 40, 'A')
self.assertEqual([], hm.retirement_blockers(
old, [], {'brainjar.net'}, {'brainjar.net', 'dead.example'},
['dead.example'], now=self.NOW))
@FIX_ONLY_RETIRE
def test_name_covered_by_another_valid_cert_does_not_block(self):
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
other = self.cert(['other.example'], 60, 'B')
self.assertEqual([], hm.retirement_blockers(
old, [other], {'brainjar.net'}, {'other.example'}, [], now=self.NOW))
@FIX_ONLY_RETIRE
def test_cover_by_an_expired_or_identical_cert_does_not_count(self):
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
expired = self.cert(['other.example'], -1, 'B')
same = self.cert(['other.example'], 60, 'A')
self.assertEqual(['other.example'], hm.retirement_blockers(
old, [expired, same], {'brainjar.net'}, {'other.example'}, [], now=self.NOW))
@FIX_ONLY_RETIRE
def test_unknown_served_set_keeps_every_valid_name(self):
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
self.assertEqual(['other.example'], hm.retirement_blockers(
old, [], {'brainjar.net'}, None, [], now=self.NOW))
@FIX_ONLY_RETIRE
def test_wildcard_coverage(self):
self.assertTrue(hm.cert_name_covers('*.example.com', 'a.example.com'))
self.assertFalse(hm.cert_name_covers('*.example.com', 'a.b.example.com'))
self.assertFalse(hm.cert_name_covers('*.example.com', 'example.com'))
old = self.cert(['*.example.com', 'brainjar.net'], 40, 'A')
self.assertEqual(['shop.example.com'], hm.retirement_blockers(
old, [], {'brainjar.net'}, {'shop.example.com'}, [], now=self.NOW))
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_read_cert_info_reads_sans_and_expiry(self):
tmp = tempfile.mkdtemp(prefix='cert-info-')
self.addCleanup(shutil.rmtree, tmp, True)
path = os.path.join(tmp, 'c.pem')
with open(path, 'w') as fh:
fh.write(san('expired_old.crt'))
info = hm.read_cert_info(path)
self.assertEqual({'brain-jar.com', 'www.brain-jar.com', 'brainjar.net',
'www.brainjar.net'}, info['names'])
self.assertEqual('brain-jar.com', info['cn'])
self.assertEqual(1585699200, info['not_after']) # 2020-04-01T00:00:00Z
self.assertTrue(info['fingerprint'])
class TestSanOverlapRetirement(CertPublishTestCase):
"""The new bundle must not stay shadowed by an older cert sharing a SAN."""
def setUp(self):
super().setUp()
self.le = os.path.join(self.tmp, 'letsencrypt')
self.new_src = os.path.join(self.tmp, 'issued', 'brainjar.net')
self.write(os.path.join(self.new_src, 'fullchain.pem'), san('new_bundle.crt'))
self.write(os.path.join(self.new_src, 'privkey.pem'), san('new_bundle.key'))
# certbot "issues" brainjar.net into live/ when called.
os.environ['TEST_CERTBOT_INSTALL_SRC'] = self.new_src
os.environ['TEST_CERTBOT_INSTALL_DIR'] = os.path.join(self.le_live, 'brainjar.net')
for name in ('brainjar.net', 'www.brainjar.net'):
self.add_domain(name, name.replace('.', '_'))
def old_lineage(self, lineage, fixture, publish=True):
"""A lineage laid out the way certbot does it: real files in
archive/<x>/*1.pem, relative symlinks in live/<x>/, a renewal conf."""
live = make_certbot_lineage(self.le, lineage, san(f'{fixture}.crt'),
san(f'{fixture}.key'))
conf = os.path.join(self.le, 'renewal', f'{lineage}.conf')
if not publish:
return live, conf, None
pem = self.write(os.path.join(self.certs, f'{lineage}.pem'),
san(f'{fixture}.crt') + san(f'{fixture}.key'))
return live, conf, pem
def bundle(self, **extra):
body = {'primary': 'brainjar.net', 'sans': ['www.brainjar.net']}
body.update(extra)
resp = self.client.post('/api/ssl/bundle', json=body)
self.assertEqual(200, resp.status_code, resp.get_data(as_text=True))
return resp.get_json()
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_expired_overlapping_cert_with_a_different_cn_is_retired(self):
"""The incident: CN brain-jar.com, SANs include brainjar.net."""
self.add_domain('brain-jar.com', 'bjc')
self.add_domain('www.brain-jar.com', 'wbjc')
live, conf, pem = self.old_lineage('brain-jar.com', 'expired_old')
data = self.bundle(unreachable_names=['brain-jar.com', 'www.brain-jar.com'])
self.assertFalse(os.path.exists(pem),
'the expired SAN-overlapping pem is still shadowing the bundle')
retired = [e['file'] for e in data['cleanup']['removed']]
self.assertEqual(['brain-jar.com.pem'], retired)
self.assertTrue(os.path.exists(os.path.join(hm.cert_retired_dir(), 'brain-jar.com.pem')))
# Lineage retired, not deleted: renew-certificates.sh can no longer
# re-publish it and certbot can no longer try to renew it.
self.assertFalse(os.path.exists(live))
self.assertFalse(os.path.exists(conf))
# live, archive AND renewal leave together (review H2), into one tree.
self.assertFalse(os.path.exists(os.path.join(self.le, 'archive', 'brain-jar.com')))
self.assertTrue(os.path.isdir(retired_path(self.le, 'live', 'brain-jar.com')))
self.assertTrue(os.path.isdir(retired_path(self.le, 'archive', 'brain-jar.com')))
self.assertTrue(os.path.exists(retired_path(self.le, 'renewal', 'brain-jar.com.conf')))
# Relative symlinks still resolve in the retired tree (restore = mv back).
self.assertIn('BEGIN CERTIFICATE', self.read(
os.path.join(retired_path(self.le, 'live', 'brain-jar.com'), 'cert.pem')))
self.assertEqual([], self.certbot_deletes())
self.assertTrue(os.path.exists(os.path.join(self.certs, 'brainjar.net.pem')))
self.assertTrue(self.edge_would_start())
self.assert_only_final_pems_in_certs_dir()
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_expired_overlap_is_retired_even_without_the_unreachable_hint(self):
self.add_domain('brain-jar.com', 'bjc')
live, conf, pem = self.old_lineage('brain-jar.com', 'expired_old')
self.bundle()
self.assertFalse(os.path.exists(pem))
self.assertFalse(os.path.exists(live))
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_shared_pem_still_serving_another_site_is_kept(self):
"""The shared-PEM hazard: never retire the only valid cert for a served name."""
self.add_domain('other-site.example', 'other_site')
live, conf, pem = self.old_lineage('shared-old.example', 'shared_old')
data = self.bundle()
self.assertTrue(os.path.exists(pem),
'retired the only valid certificate for other-site.example')
self.assertTrue(os.path.exists(live))
self.assertTrue(os.path.exists(conf))
kept = {e['file']: e['blockers'] for e in data['cleanup']['kept']}
self.assertEqual({'shared-old.example.pem': ['other-site.example']}, kept)
self.assertEqual(['other-site.example'],
data['cleanup']['lineages']['kept'][0]['blockers'])
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_valid_overlap_whose_other_names_are_unreachable_is_retired(self):
self.add_domain('dead.example', 'dead')
live, conf, pem = self.old_lineage('dead.example', 'dead_old')
self.bundle(unreachable_names=['dead.example'])
self.assertFalse(os.path.exists(pem))
self.assertFalse(os.path.exists(live))
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_valid_overlap_still_served_without_the_hint_is_kept(self):
"""Same files as above, but nobody said dead.example is gone: keep it."""
self.add_domain('dead.example', 'dead')
live, conf, pem = self.old_lineage('dead.example', 'dead_old')
self.bundle()
self.assertTrue(os.path.exists(pem))
self.assertTrue(os.path.exists(live))
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_non_overlapping_certificates_are_untouched(self):
live, conf, pem = self.old_lineage('unrelated.example', 'unrelated')
self.bundle()
self.assertTrue(os.path.exists(pem), 'retired a cert sharing no name with the bundle')
self.assertTrue(os.path.exists(live))
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_response_distinguishes_issuance_from_not_yet_due(self):
issued = self.bundle()
self.assertTrue(issued['renewed'], 'a fresh issuance was not reported as one')
self.assertEqual(4922899200, issued['not_after']) # 2126-01-01T00:00:00Z
# Lineage now exists with the same names; certbot keeps it.
del os.environ['TEST_CERTBOT_INSTALL_SRC']
os.environ['TEST_CERTBOT_NOT_DUE'] = '1'
noop = self.bundle()
self.assertFalse(noop['renewed'], 'a certbot no-op was reported as a renewal')
self.assertTrue(noop['certbot_not_due'])
self.assertEqual(4922899200, noop['not_after'])
self.assertIn('--keep-until-expiring', self.certbot_invocations()[-1])
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_not_due_noop_never_retires_its_own_cert_name_lineage(self):
"""Review H1: find_certbot_live_dir() returns the NEWEST of brainjar.net
and brainjar.net-0001. After a certbot no-op that was the stale -0001:
it got published as brainjar.net.pem and the real brainjar.net lineage
was retired as "superseded"."""
del os.environ['TEST_CERTBOT_INSTALL_SRC']
os.environ['TEST_CERTBOT_NOT_DUE'] = '1'
real = make_certbot_lineage(self.le, 'brainjar.net', san('new_bundle.crt'), san('new_bundle.key'))
stale = make_certbot_lineage(self.le, 'brainjar.net-0001', san('dead_old.crt'), san('dead_old.key'))
later = time.time() + 60
os.utime(os.path.join(self.le, 'archive', 'brainjar.net-0001', 'fullchain1.pem'), (later, later))
# Production behaviour of find_certbot_live_dir(): newest wins.
hm.find_certbot_live_dir = lambda base: stale if base == 'brainjar.net' else None
data = self.bundle()
self.assertTrue(os.path.isdir(real), 'the --cert-name lineage itself was retired')
self.assertTrue(os.path.exists(os.path.join(self.le, 'renewal', 'brainjar.net.conf')))
self.assertNotIn('brainjar.net', [e['lineage'] for e in data['cleanup']['lineages']['retired']])
published = self.read(os.path.join(self.certs, 'brainjar.net.pem'))
self.assertIn(san('new_bundle.crt').strip().splitlines()[1], published,
'published the stale -0001 certificate instead of live/brainjar.net')
self.assertFalse(data['renewed'])
@FIX_ONLY_RETIRE
def test_bundle_reports_capabilities(self):
data = self.bundle()
self.assertIn('san_overlap_retirement', data.get('capabilities', []))
caps = self.client.get('/api/capabilities').get_json()
self.assertIn('unreachable_names', caps['capabilities'])
class TestRetiredLineageCanBeReissued(CertPublishTestCase):
"""Review H2: retirement must leave certbot able to issue the name again."""
def setUp(self):
super().setUp()
self.le = hm.LETSENCRYPT_DIR
@FIX_ONLY_RETIRE
def test_retirement_removes_every_part_certbot_checks(self):
make_certbot_lineage(self.le, 'brain-jar.com', san('expired_old.crt'), san('expired_old.key'))
out = hm.retire_lineage_tree('brain-jar.com')
# certbot new_lineage() refuses if any of these exist for the name.
for part in (('live', 'brain-jar.com'), ('archive', 'brain-jar.com'),
('renewal', 'brain-jar.com.conf')):
self.assertFalse(os.path.lexists(os.path.join(self.le, *part)), part)
self.assertTrue(os.path.lexists(os.path.join(out['retired_to'], *part)), part)
self.assertIn('BEGIN CERTIFICATE',
self.read(os.path.join(out['retired_to'], 'live', 'brain-jar.com', 'fullchain.pem')))
@FIX_ONLY_RETIRE
def test_second_retirement_of_the_same_name_does_not_collide(self):
make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key'))
a = hm.retire_lineage_tree('x.example', stamp='S')
make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key'))
b = hm.retire_lineage_tree('x.example', stamp='S')
self.assertNotEqual(a['retired_to'], b['retired_to'])
self.assertTrue(os.path.isdir(os.path.join(a['retired_to'], 'archive', 'x.example')))
@FIX_ONLY_RETIRE
def test_certbot_can_issue_the_name_again_after_retirement(self):
"""Against certbot's own storage code (skipped where certbot is not
importable, e.g. the image's app python)."""
# Import AND initialisation are both guarded: a half-installed or
# version-skewed certbot can fail with far more than ImportError (a
# missing transitive dependency, a pkg_resources/entry-point error, a
# moved internal). Any of those means "cannot run this check here",
# not "retirement is broken". new_lineage() itself is NOT inside the
# guard - that is the assertion.
try:
import types
from certbot._internal import storage
real_rv = storage.relevant_values
new_lineage = storage.RenewableCert.new_lineage
cfg = types.SimpleNamespace(
config_dir=self.le, renewal_configs_dir=os.path.join(self.le, 'renewal'),
default_archive_dir=os.path.join(self.le, 'archive'),
live_dir=os.path.join(self.le, 'live'))
except Exception as e: # noqa: BLE001 - deliberately broad, see above
self.skipTest(f'certbot unusable from this python ({type(e).__name__}: {e})')
storage.relevant_values = lambda c: {'server': 'https://acme.invalid/directory'}
self.addCleanup(setattr, storage, 'relevant_values', real_rv)
cert, key = san('new_bundle.crt').encode(), san('new_bundle.key').encode()
first = new_lineage('brain-jar.com', cert, key, cert, cfg)
self.assertEqual('brain-jar.com', first.lineagename)
hm.retire_lineage_tree('brain-jar.com')
again = new_lineage('brain-jar.com', cert, key, cert, cfg)
self.assertEqual('brain-jar.com', again.lineagename,
'certbot could not re-create the lineage under the same name')
self.assertTrue(os.path.getsize(os.path.join(self.le, 'renewal', 'brain-jar.com.conf')) > 0)
class TestExpiredCertificateIsNeverPublished(CertPublishTestCase):
@FIX_ONLY_RETIRE
@NEEDS_OPENSSL
def test_publisher_refuses_an_expired_leaf(self):
src = os.path.join(self.tmp, 'expired')
cert = self.write(os.path.join(src, 'fullchain.pem'), san('expired_old.crt'))
key = self.write(os.path.join(src, 'privkey.pem'), san('expired_old.key'))
live = self.publish_live_bundle('brain-jar.com')
with self.assertRaises(hm.CertificatePublishError) as ctx:
hm.publish_pem_bundle(live, [cert, key])
self.assertIn('expired', str(ctx.exception))
self.assertEqual(PREVIOUS_BUNDLE, self.read(live))
class TestBundleValidation(CertPublishTestCase):
"""What may and may not be published."""
@@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC9DCCAdygAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMZGVh
ZC5leGFtcGxlMCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAwWjAXMRUw
EwYDVQQDDAxkZWFkLmV4YW1wbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQCumL3T27a3D6fZbIgwovdH6SfvKAkz3NwBiDLTtIn+Zia3l5FynrGXWZnO
abvnQNxEVqyYtTJOn9taitVc30wLVpk31BQdLab5rdVa4IADnd3Mel5Cd0M+P3BU
SpW96rK40KVftzzDEUnH+3ncuzPQ7sKXom0JYqBXT42s8LF5UKtXI1oF1iW/7uz0
seOIdyP/n3T2cX1wsCa/F0v2MAyze5FhdtWij+2IK1FRbIbSSr6zUCAeAWdqiM9b
PBd2cBaBwTe/3TICu0RnvvRBvlMNrgbEBm+wnDf2qlxAO2pSER1bipi4C9gu1nm2
wzG4zlKU31fdLvQOzMGH1W5H91sBAgMBAAGjSDBGMCUGA1UdEQQeMByCDGRlYWQu
ZXhhbXBsZYIMYnJhaW5qYXIubmV0MB0GA1UdDgQWBBSPDJiv0Z2YpuloexOfvpIQ
UrWufTANBgkqhkiG9w0BAQsFAAOCAQEAJwBtN3Qw41XvYuSa8Mjn869QOTKEgkaR
EuNQ8WXVBOTRJdxaLHDunGO4ExjHHswCWzXkefYOoIClAU9qLnk9M667BSNxOcz0
VcG/DUMd0tO8X4Zm0zOMjt4VC7whgj7MdYtYVg2+Jhb4sMSx6OqmCUAD4X6ULSTn
lZK2holEC/H3bKt8Q6cecgeubgWojP1bWmWK7jQL5NRIzxQ0KzFxgAf3t2Z05nX8
loABkBiR7vDthkQseUW6RQMayDyzhVqORLt+gZ9D5T3hT1M7VaDc4Twr7UbWPFcR
GuQ7RwKplPIIbPe7+c91pm+agdSDjnV5HmKcAMCwf7bWGQ9/Xc0BFg==
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCumL3T27a3D6fZ
bIgwovdH6SfvKAkz3NwBiDLTtIn+Zia3l5FynrGXWZnOabvnQNxEVqyYtTJOn9ta
itVc30wLVpk31BQdLab5rdVa4IADnd3Mel5Cd0M+P3BUSpW96rK40KVftzzDEUnH
+3ncuzPQ7sKXom0JYqBXT42s8LF5UKtXI1oF1iW/7uz0seOIdyP/n3T2cX1wsCa/
F0v2MAyze5FhdtWij+2IK1FRbIbSSr6zUCAeAWdqiM9bPBd2cBaBwTe/3TICu0Rn
vvRBvlMNrgbEBm+wnDf2qlxAO2pSER1bipi4C9gu1nm2wzG4zlKU31fdLvQOzMGH
1W5H91sBAgMBAAECggEADUvYBFLn6gPWSZj55nxOqR+2Lmtwh04omgbu2fpWhPTE
ujgKd/23srnnMZXnR6qZQQIPGwJiOPnJuRjiu/PMZSj8j54bbwfA6XmbH9SCIzm0
la50UGBSkw0T50jBI76vICMm6PBkf0rT1oKN9VOd/qlkJoCUiFcGCh177Fbt07oY
R3iAKu7/+yEANX0iEcxbO330TPW3oI0ah5OryfKVoNvxpF8NUfV9pWY66APkOYMJ
zsYvIMvzV/KIgfrtyxJjSjgd7Ud8uictYG1BrZBXpIuGxuxyKUJ7OXOvMMVNnEME
2uzk/aL75UjpurJJU0LCywdUOGwRqS2q4PYQiTeDLQKBgQDjdywH7RODoYeH05Rd
sa2MkNx0Day1i3A4IBds55mzn8jxpT55HlqvHWhNA30kEw01H8+6sB7iMX04xQtS
u3N2opR1VlCvMJhGyRUYUtfu4NnAi/HSu7mCOFGAwaLclI7BAI2or2UrclrlUhGb
CYIBKfEr6Tn6b6AffWaOruCbtQKBgQDEf70R922fnJK+//QuiYoaESwJd3/k10yJ
nH0ZAgbnFcNLT9YVFxHvlt4RHGIotaBjfN+FKMn045jeekgeVdX+uF8+7FwEazAH
zOfxswOeKd8EOooqCD7n7cW3Ma6ev8oWmXwJsz2coRW7Z5zCGQWSzCEDB4yFU9Mt
0z6aN9SJnQKBgQCvBZSIqL+6KgD53zArA7sRDKnEVn1M13TKis7bij02/DyGyuoO
EfAfDBakI4oXxt0Dwm15LWkmDIRYuS0kRH3si96FhpvgoRiaZsDnjV7/PF3wjP3O
c4mjDCiSVOw2o8xP71gQspy62GveRc6sJ52zvY0Hh5aH/3oKoNfX8/F5fQKBgALQ
vbnDLMvrp2KbMjl0eQkUk+fy0V9LAmq6VkSEderbsfejRjL5KKWhKL9gA7wHliu2
v1EeG36pwCf34o6ktjM3Gw5AvrTFgGOsOOqKoROkD/D0kcQ2aAhUrM+KI7FcmB2o
x1EwQ0K6QCB2RJW3XBAVYqezdPz4DX+LlksJX781AoGBAISGOer3s4+oRjO/2VuT
hnMQrET3A0GTol8cgYL2N9wfx7jy5bhSfiJIX/dkh5IrPGYoBdWM87gcUgNQ8AZ7
il5dVeG80oc7KFU/QltZJu9VS32vwXnNsnrXupQ5Ee0ztTLciTMWU5jxvB0ZJHtK
StgYepr4IPDUaObqrPx4Jvfm
-----END PRIVATE KEY-----
@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDGjCCAgKgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwGDEWMBQGA1UEAwwNYnJh
aW4tamFyLmNvbTAeFw0yMDAxMDEwMDAwMDBaFw0yMDA0MDEwMDAwMDBaMBgxFjAU
BgNVBAMMDWJyYWluLWphci5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQC9SJUMsK5HhVEATu/FWwJsvaK7lkigh7EhlkxTNkLFNzlgkLcoKMhD2F9s
i5KzgYp0K2z899wQAKuLzZcde6BUnw6RHzheCRsIqLZlbY8B3Ayd6VhSMYqhWoXJ
7jax2QWFMQVhyy6M5QTCHUVJACMpega+KrsYuNkNAxLc2uGhgHLPnnOoDhGXosyN
N8eolukhvFxQNU3E7oo1jBzzzSN+FFPCZ8oWab6ydW7YSxZLlxRWoQ+2mL9I+SSy
UEc+Gje6dJLZ9CdJtSQSNGnFoqOUGIipNpS2rrBGctVWijM9y459skTFKbamLPy8
FHC7atJOe5EoaekaAXw5Tq+yB+ODAgMBAAGjbjBsMEsGA1UdEQREMEKCDWJyYWlu
LWphci5jb22CEXd3dy5icmFpbi1qYXIuY29tggxicmFpbmphci5uZXSCEHd3dy5i
cmFpbmphci5uZXQwHQYDVR0OBBYEFAWqbBh2PDUvqQzZoyh960cpuhcOMA0GCSqG
SIb3DQEBCwUAA4IBAQCaovtrXl56YcFlXWMrW/XZBe2/fXI14860qyiR5rb07lS6
CbKOsnRyQdxZ3cdfRwTN1UMmz5IPOZ6fhdo3IPaS2Wae1OdeupRaRWr1SAJtpVZ3
WWugDaumsWV5eWVxVTf93yDEkMERWtTGMviCKBeeHjzHIFiD/EjZznE87V5DYwuf
guOd1F4wXCMRFu8q7sBdPXjQ8rZMlQT+dQXzMST9Ee5PXiInmCAA5nT8vMInAwd2
rP6FxuHJB75ekYzs57CJ82BKKboOCESjUbm54wmVV5pBL305/EjxGf+VovNlRktg
t1znolntIIofE25UCbVNPdrtuDkZgKT3Rv8tCdTP
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC9SJUMsK5HhVEA
Tu/FWwJsvaK7lkigh7EhlkxTNkLFNzlgkLcoKMhD2F9si5KzgYp0K2z899wQAKuL
zZcde6BUnw6RHzheCRsIqLZlbY8B3Ayd6VhSMYqhWoXJ7jax2QWFMQVhyy6M5QTC
HUVJACMpega+KrsYuNkNAxLc2uGhgHLPnnOoDhGXosyNN8eolukhvFxQNU3E7oo1
jBzzzSN+FFPCZ8oWab6ydW7YSxZLlxRWoQ+2mL9I+SSyUEc+Gje6dJLZ9CdJtSQS
NGnFoqOUGIipNpS2rrBGctVWijM9y459skTFKbamLPy8FHC7atJOe5EoaekaAXw5
Tq+yB+ODAgMBAAECggEABAeOCL71eYH4jKRMOBbXu98483u1CxnTEa8klnw9ZVES
JctJjY96B/EMwv+g/V+gxhgAkiK9VWfg6bEplOVDWgR9fRBoP57mVkOv3QPn5fcR
xlt9LQQZ+pULQRZoUAFEkVEkV1sfIBJmDLMufOnbAHCKa9+hVlrlCqu0lJgM8put
2bkpSAqwroHVfmD0SmKFAZggBhFwXFJzwmUf3WL+N25f4RZpJsX17FOJ216mlhQ5
edje6dcDpmko2VQtGJXYr8Ge02ZYeRyKGcTsgUZlarB4xUZq1pcnUUSy0Ez7LVbk
tKjPcUpZLlnAA/a2pDgHmwN1+fv/X+Wwv/vZaQMBTQKBgQDuZFyfDRSHCssHtQ4+
Ezxg4WEjwXkq58WxL0zTtLTcquZfQZ1VXgLwAylVwver0I4/UgCfW/AYY+2eLrba
FhuYOosd/3rsUQqf9y+TkSkhe1uVS2/1zjzC1G/5coleLddjzd+YBpEtVxfxDRbA
rVePY3GxkJiTrjAg5OETk7GYpwKBgQDLQ6bPhaocJPQmDi3hKyW8zPJ8wAJNyI0c
RWt9gjeidjmMvwCKtJbBDcV5V0oQ5SB2M5E49mT58B62totAD7FS18c4a4133haV
a1pLBX0U524eQgDzMJ6YA6N9bHdXOH4FZd+Pw90bHetbT1HcKpOdXv4SF/XITju5
Ro5xiymdxQKBgBB4fq2+iZyK8lwl9kXzrkhDSiLeSzDbspz7V6OI6LYWWLRcJDKx
vCI5FbIHNnV4g1t8nu5ju8LzpIc5UwjeXim2YcZA0I1jTAihW3gmn2Dg1QMLo0Co
bypG4zWo5DTNlgQfRUOLsJwhWFYfQWMuls4hYMmT4/bQIiMjrSldGvghAoGALpaB
7DyZobvCjdpgH6TREHX0+izUGlnmJd2vZmPrNnxccmrwx9neV789TLN6EWdCynTF
S0YGHNcxYvVx8yvlm7RDQCY+lYO8r2CcoGdQpPLqZVsBdrvlDbh/x2aN79rN35Ta
6cudqPJxmEsU2x6TGNXF7AW72r47qBqI2XNpBXUCgYAFQdHRRxa2E5rgV0WkL7PF
vXok3isAzpnKAsb5oxPZbhIdLo5EPUGCslsTE1Na8NBdUSoyjhgkMELEDx/ceGL7
IIlNrc/4j71+IahljzaX/D5b5qaW+RnoTVAlbMRR7L/ecbkoovhQR9d37QdoiD/t
B9si92+XB4HYDAjXFxTK3w==
-----END PRIVATE KEY-----
@@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC+DCCAeCgAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMYnJh
aW5qYXIubmV0MCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAwWjAXMRUw
EwYDVQQDDAxicmFpbmphci5uZXQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQCyhL5YcFhPEekc/xNMaEHO0hjoNRd5yNhG3BF+pbChYJhkjw5McdH89fdb
l4twhlkcyvyaEbcHIgLEgz5caANrlGnzQBTOBTILhA0leTF9hBCQX1yKEotQFdgU
QcAAvdAyP77jDF6lX11oCjJwzoT0YeLcFG3XyfDQ7hRSJNJFDqvbOgCc9kZ+B3zz
XRVp4l2WyqEUJmKxcpppAnfbVhokwy2417CumYp8wIzYJ57mRk6RrRRNbENw3wGU
0v78QfI+ueCw261UWZ+iXwlZt/icvzWtcuoPVxid3uPeRPGV40RUbPGuetvIjLNQ
dFOfKjH0QGInyByji9eAaXlmvCKtAgMBAAGjTDBKMCkGA1UdEQQiMCCCDGJyYWlu
amFyLm5ldIIQd3d3LmJyYWluamFyLm5ldDAdBgNVHQ4EFgQUdDxfZKX96j61mW7O
NiQpN4T+sZ4wDQYJKoZIhvcNAQELBQADggEBAJE1lP62caByZT7GkKtJLfOj84Ih
MeP751w2gdU8ZfiO8GxXXiR3ZjYzbJgsEi9LiNT9jqHeBf3pv/mJLjPVZ+U557ws
Gq3V5A3uKlRKGcejXlHaWnExhWqLyIaLTeLKstbJyiB/ja4yeTL2TYcOvvlu6weh
ERqwx0p6r7LSQ3nRfTdjEhNq3FVUmqQ8neBP/vXLp2hKr2ACZndGyQOiv0oZWzud
part6RMd8gRNq1Tctrl0YqRYoLRIddBrnTtUtac4FLqxLSBNc03/I9ZgF56Yf+Sl
M1ikmyDJ2ZYJFU6v42Og4/8o3fpq0ttP6pKPi8fInGaFQ+KXO1NWFf1ogD0=
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCyhL5YcFhPEekc
/xNMaEHO0hjoNRd5yNhG3BF+pbChYJhkjw5McdH89fdbl4twhlkcyvyaEbcHIgLE
gz5caANrlGnzQBTOBTILhA0leTF9hBCQX1yKEotQFdgUQcAAvdAyP77jDF6lX11o
CjJwzoT0YeLcFG3XyfDQ7hRSJNJFDqvbOgCc9kZ+B3zzXRVp4l2WyqEUJmKxcppp
AnfbVhokwy2417CumYp8wIzYJ57mRk6RrRRNbENw3wGU0v78QfI+ueCw261UWZ+i
XwlZt/icvzWtcuoPVxid3uPeRPGV40RUbPGuetvIjLNQdFOfKjH0QGInyByji9eA
aXlmvCKtAgMBAAECggEAEQi8lvjFXGojmco3H7PVD7SZftLOFidVh9Fo8/JAe+2Z
LQM/hIjZ1i7sNverbQXMzAQMfXESS2Jx7Clv0mZ0aRgE0hq7wXrabyHfFUPOtqDa
6cPHsz8YvwAOmcQ6W3fBPYI6pzSfdV03rv1GGeCC7ETI3mIupwgUbANDuiDcVY1q
+VuK/hDzPJ5KslDHUNR42MgWacIftq99fV9jN0zQQw1EcyO8QwpXzTSEbdyiAkT+
n/tR9He2MYhgRhjyWPMaxmY6XQpv1h5ASq1yT1kI2oNTaRT3xvd40sLkbfwpZnxi
HnqzCkPptEpWeCLrme+r71+Hxu/+Bph7tPtcuK7jgQKBgQD8XCoPRiWbgCl6kM2G
E3GeM6fftTieTIs7sFVte7wutIKlsec4vizc94ipGaoT7yGEG0T8g9+5hAMbebc2
zzhxvJMB2s18Z/He+Ujv1L1AhBBNVY3zCAxfdq8/n5MGXppEdVRo9QOQndoBfuAV
Qw2Vp4bcy728n4T7sBugW0Ad7QKBgQC1F+utuiF8XI2yx0mX0+Fbty6o+odyM0eb
G/QQDlcGpY0K7V3WAFMESNfytig8OhidzRfWn8lNoqhd80SJfCuiYjiGJ8tjBzcp
+Vqrd/KAxJySJ3NJ2MwyAp7W3l3HAEZLGVYVCankoa4O3qR8YqaIZwc2nk5+8KnG
nBYFlSF/wQKBgQCqaRYh0E2O7wGTeXq/ohukJ9+hqf88rhWLTeyyDM8rfsuGVlb8
MzeaUzEed20pHt60FbIFDowh+yoaAJwez9mevCdNJlKCQrLKze1Zvn+5tDRJ9I7x
YSrxv/Rhc0FWGMX1moFQirtIib9wKRvqX6gpucHREWfjmpizWN2e8F+glQKBgAOp
/jy/nVnUzZEmcxi9Urv2Ut1cZE0tOG+s+x0wDRa3Vsk30EQ6MXPOvX/Nq7a8Pi9D
LQ5MoVgOyHVOLL7TYlagwiAEwhq41KbmwqzhemickcZrQevmlQsV/ny+dwtFuCZg
1tsKHQPhOZJXcuCJZnEwFHdIcLIk0GqSUee/ArSBAoGALOD+Opk+0j4ObNcqtrpP
UcEYqhOuAscbxLoUv53wnOvc7XzMG8JCK1y5LthB2Xanr1JSlheMRrvGxC+WlDmc
5v0Nz71cq5QQ1p8AXnosLRjuy/+edMvGIxxwFMEJBO2ra2Jizpdwd34bjpQ42+CQ
6pVwFQX0oet8SI5xWUWx7NU=
-----END PRIVATE KEY-----
@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDGjCCAgKgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwHTEbMBkGA1UEAwwSc2hh
cmVkLW9sZC5leGFtcGxlMCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAw
WjAdMRswGQYDVQQDDBJzaGFyZWQtb2xkLmV4YW1wbGUwggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQCBg7iYuvjhIG7nlFeja5Lyv2t2fdtNvxzUib7ygejC
+WQvNdH4k12q8vCXNYuadbGAceSejG4bazpjo5/ym2KFZPwiSZoqKUugi9lCoKuL
2EUmCHoTsr+CLPuZXyjxu/x250m5llJOhgMHYFwURjARRAku3a/GJUHQR0C572mW
mt/WtWueHlqEFoI7jCuBAUmww1p1YNV0cl64fGeYSOaYOHEOp/VqIRlzsy7eGQh3
xLQzEMyXzrjD54PZ6NuJeT7heZHVO3if2JbqFxvijCLnBz5WsrS/S/VWOELNEfGr
84/Jcwze9ZV8e0y4tOwh9qT82adbl1nyAgLfWuSgp5hRAgMBAAGjYjBgMD8GA1Ud
EQQ4MDaCEnNoYXJlZC1vbGQuZXhhbXBsZYIMYnJhaW5qYXIubmV0ghJvdGhlci1z
aXRlLmV4YW1wbGUwHQYDVR0OBBYEFP+AdZ9MkBmFjAQTgLnWZxbJtnvkMA0GCSqG
SIb3DQEBCwUAA4IBAQA32Xl/1IrOyESd2UQcWp/fuHvkivLgamJrheKvKP0Y0vMV
atRu0D0GhqMOr79OT60H/H1FFW7bWQ+UPcZpnuXec40/PcrHxwaSoiLoIUVWhB4t
BOhtTDReX29gEjnB7OqBCJu4ub3HnpFTflUZd7n5A4EcetBsowc5EZ9UAtfvkgGD
nK9ONvNVGurk/V8xgtMJ877ZK1fn+UtgvIWp/F1Gc/taLhe+oCZOPkOqvvqUnS8q
QyIhqmMWRUkBP+jwRA4TP4pj7OH38VY/Ae408Xe4Y97GF7NbOjAL0bkX/bHkLnSG
WzoGO0TK7pCemeFQ+Ghic8Yl0eDynoP7rZIhCKS0
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCBg7iYuvjhIG7n
lFeja5Lyv2t2fdtNvxzUib7ygejC+WQvNdH4k12q8vCXNYuadbGAceSejG4bazpj
o5/ym2KFZPwiSZoqKUugi9lCoKuL2EUmCHoTsr+CLPuZXyjxu/x250m5llJOhgMH
YFwURjARRAku3a/GJUHQR0C572mWmt/WtWueHlqEFoI7jCuBAUmww1p1YNV0cl64
fGeYSOaYOHEOp/VqIRlzsy7eGQh3xLQzEMyXzrjD54PZ6NuJeT7heZHVO3if2Jbq
FxvijCLnBz5WsrS/S/VWOELNEfGr84/Jcwze9ZV8e0y4tOwh9qT82adbl1nyAgLf
WuSgp5hRAgMBAAECggEACB1tvTwciKvp5wliGF45feDv5c5q+vXE462h0OYXjyfl
1sNuMuoLjhYyFvdcQO2rlTBjB8Ald1hXoeidsVYlF9XvqmJRjjS6MAazMpEzTnba
66V6U1j5gOQ6pi3SjfBJGFYo8L+cy/NqUIBWz0+6hp/BsjW3uW2aMGer2Y/zs2Oi
s0kUNqqUWQpGpEDxbDnUAzto4OOcTOO4A5NB5Oe2ifzFiXCWtTNqFIC0ae9tiqGy
m0sggF2hzmlH1WC54rbv7xujYT6tOAEi1OKp/Cpj8XVLbjaqju5avRl1t2nDRQri
9vnOI9P4bnJKbIYzwJVyy8B1M5iK+W2K7wUQ4ZMetQKBgQC2zwtywQ4jUduLWf4f
6HLUIvmcHdJI0YBHS1+4fKc7tsakRo/0FHxNdUlBhqkYTyp57h6/R1T2gG70Kla1
MarGIgyPIlUXsRojlUcNyKXqSVs+kejftIClb+VDXWrpKtGtyxuESsMjaJBsBBAN
hg3yUx2NL2ct4nnlpl1iq3pH9wKBgQC1XkrjntuzjVYlFeg2ih5va3PUD2qmH9Ej
leNXTGhzZL1eQWz4GaRCgzJH8q41pXBxcXQ8hMXZCWdW5GHBYfVqjTRKMUPdI6FI
ypwMbbTBfQfw1kz5VHfLH3romxikNQXeorZkIZqv7/aLPsxLY78XdW8nSDIsT9ZP
mh7It+Tf9wKBgHT7VSYf6D4t2cH42qfshd6+auC+LJywg9gboatbCWEq+z6QcPoO
SeT/Utt9oqONtW7FNCPPL812LopMtaPJnrCfhU3q+GzqxgFTO17UsWeVfbAyisTs
qw4y5pEPLKT846TFPlyPwwPbm9+jUep1iGMzUMvSM8KnFL4ikcvcU6nNAoGBAJQI
MVHV0HadzokYd19EnpDMgE6+goqy/21vIo6XY82DlPjAp+NccnCDSt/kmQJaVQuS
0Y+vwxhhj4IhiBRiEwBz3hEMYtq1ASg7juCqDlW66mdNUqhSpDAbMqN2K8ejK3ZL
EllqACrHzlBa4j5KXKcWB7JlP+ZhB4gTQDsNtXaDAoGBAJCktIdyM3b6UtM5OofB
lzn6/2uT5u46ECBRybk/J+Y2gfcS2cFzkH179SUB4WgfR0h21FlLmjns5ZvHVk0p
/U4aY3/J42JU6lCYrsyV5BBH+O0stSmrUG/4JdKRe6Mq/pB321RsWD06LVJrBDpi
6NUPW03tNLc+Qe+cnQu0c7y+
-----END PRIVATE KEY-----
@@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC8zCCAdugAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwHDEaMBgGA1UEAwwRdW5y
ZWxhdGVkLmV4YW1wbGUwHhcNMjAwMTAxMDAwMDAwWhcNMjAwNDAxMDAwMDAwWjAc
MRowGAYDVQQDDBF1bnJlbGF0ZWQuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQAD
ggEPADCCAQoCggEBALEpfqdRPBSrbzixAxD2WhsO1AOTUd/Gguf3zjDZ3TUmWJKb
OhlFthtU6PJVhMdAqOk+hiz9Coo/UngWjykyNMOj1YIpq3M4/P8wBTNxkJd/z2x0
iAQBKvWlB6hxf9c1kenu/ea1iMIThA/uudtX3g8k39Asj/37+o070cdBJExWGnbG
lz103FCO6kBzlJPGIVQwzKH6OKthTzi7Uxy5Fxc/ymS9gc43F0tyY6892Wyl8Y62
a3r1PpZoWR0GDFxPIyCS7NP44T15DSL/2RBi4YnWXOC8QMvPnHbSKjoQ6GLQtGkO
1NO4Uy3XbwDFanYXv6Wxv0oUmZMuY7CRTOMdxCMCAwEAAaM/MD0wHAYDVR0RBBUw
E4IRdW5yZWxhdGVkLmV4YW1wbGUwHQYDVR0OBBYEFBh0L/5U5f94TylibBCgS9pD
/VXkMA0GCSqGSIb3DQEBCwUAA4IBAQB5MLqNjTqIdyztjJsKPdYUkA27NpfdqqY6
Yprv5N7U0slsGUqFIDrSuJ1mMMwTpt02IAFyCOWMUtze+/0PqM9k9TqxJH2cTODp
0ET34Qh8NTo0B534EQUWhny+EzgNAzWK02wbpgm88+Mp/t//sHzhWzOm7uk1dlkq
+5ty/8OveWwAtqAWlDtcsRhkEjzynZKUCAF1extwUSuapFSm20mJ4ivt7Ff5MkTI
fq8P+3XAxWTaBW8DzBIzYd1mwFHK2pcklLMRRTW3hukPxEcTivBgJdwsHMskM3UO
/ejy5EHem5ohCMbwMybvUqLS4xdoqXV36Sj2I1zv0IjgO0wgdK1r
-----END CERTIFICATE-----
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCxKX6nUTwUq284
sQMQ9lobDtQDk1HfxoLn984w2d01JliSmzoZRbYbVOjyVYTHQKjpPoYs/QqKP1J4
Fo8pMjTDo9WCKatzOPz/MAUzcZCXf89sdIgEASr1pQeocX/XNZHp7v3mtYjCE4QP
7rnbV94PJN/QLI/9+/qNO9HHQSRMVhp2xpc9dNxQjupAc5STxiFUMMyh+jirYU84
u1McuRcXP8pkvYHONxdLcmOvPdlspfGOtmt69T6WaFkdBgxcTyMgkuzT+OE9eQ0i
/9kQYuGJ1lzgvEDLz5x20io6EOhi0LRpDtTTuFMt128AxWp2F7+lsb9KFJmTLmOw
kUzjHcQjAgMBAAECggEAJjBKo+iH6fC2beMCRTLFY5XDr+2etX0GFNi+8prT6zQT
3a5mTLJROf0Ipcw3bfgLMRQQOjxlfD2AGnDKdVfkNdPoH0G6OoErUBra1CcDwxSC
8t7/vtU9nGvxhabJputeQ64HvyacjzFDw2dzWsvLCzqooCgq9dXOQYmGg8wP9gAx
pxfKnd84h41C3pXERdufi6nJHPjZPRW+dKsQ1NMLmtgO5YQnTt3LfVOkZaRa7Izq
ZFWZCBfdsNHHuXCYj3FtqYrqSxELuZAxJ62hxpizrfQdKixguGTObHhC4muIo9fR
eIMCvrAzrs4Fs8nM/K/IPyDU9wPwINK6Z+sxJkhM+QKBgQDaL3Yt3T/LE6bTYyb0
EjVcXoybnuAjFQAenCTIsBRJv21Yv2ddqjq6rOTD64hFuFWPMEOi1TJxIPsFFDMw
NeAwF41fiSrtU8awXs0hTKRQeHjElSBckumOl1DrHu2oKuwWXLniViD62vMsWmpZ
89KT2rgFQuMPvsD3pps8l6W2HwKBgQDP3eTwdVJTdRYlZonqd0+LHG7h715TT1OE
QJOdN9vK5g/9GOCOGHoUxB3IxJVmvz57PrfWaD2rSZ6JMs4TcAY5a0PbJEpgqoJY
kIm/EDnSrFxu2L8DNsAzuThfzXrwWc7PzajYJctYtEfvxCyfo/GdZahg2xaEU0Le
dp+7fxtJfQKBgGVdLWtLYarH+3Yzl1JG71F1f5+A5SKHTCHsas5tt083xuY3Lh9+
yYAWeMVotEunwuAeDR4mLCQQrW4Dn3BjZz8QghaDVFUj2afwsx4oTCIyhfDWU02O
10DunaJOIR1Cq6IL/DPhzbqdY+IUM1qNAyS7tUiEdZbICUBEv9Qbo9oDAoGAORnY
cSGZaumoGsAPNMRvqmQ2JYSO4c3RwGvI4RBeJV+LlthFn2Mj7h5tnCA8dkwkytBQ
yeo6vpnrnf7B018bppxLuKBAekt8oP2biUvYxoGopp6N4kEhDVRdIxwcIHhbMoRp
VG8VbBmiBUBbjcMIdVdeqLSme9UvclMdz/C7uzUCgYEAzD2dvoJSu9zfo3fACw1X
te53Rvr6Kq6DTw0SS4UrAnxlhX9TwD9gymIrR9rxnC+UPwfMwWMdpYqY+Da4sF1G
uwd+nyZ4iFNq+lbLii9rmJ58h03+uo4lUXWTrAik6qOF1ztIfYKWFrOyBQmxjWwo
HnUinudalENFJYZRwSUAtXo=
-----END PRIVATE KEY-----