Merge fix/ssl-renewal-durable into release/2026.10.3
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
765b2ce00a
18 files changed
+1315
-114
No files matched your search
@@ -12,6 +12,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
- **Certificate destruction safety**: `python3 scripts/test-cert-write-safety.py` - offline; asserts a live `.pem` is never truncated, removed, or its certbot lineage deleted while any configured domain still references it (one bundle serves many names, so `ssl_cert_path` is routinely shared). Run it after touching any `os.remove`/`certbot delete`/PEM-write path.
|
||||
- **API thread-pool safety**: `python3 scripts/test-gunicorn-socket-timeouts.py` - runs REAL gunicorn (~60s); asserts a silent/half-written client cannot pin a gthread pool thread (per-connection socket timeout, `HAPROXY_MGR_CLIENT_TIMEOUT`), 2050 never-sending connections are reaped (`HAPROXY_MGR_FIRST_BYTE_TIMEOUT`), slowloris header drips are cut (`HAPROXY_MGR_HEADER_DEADLINE`, counted from enqueue), 40 reconnecting drippers cannot starve legit requests, the :8000 peer filter (`HAPROXY_MGR_API_ALLOWED_PEERS`) refuses non-allowed peers, slow legit requests are not cut off, the wedge watchdog (`HAPROXY_MGR_WEDGE_SECONDS`) replaces a fully stuck worker, and an in-process call to a stalled upstream times out. Run it after touching `haproxy_gunicorn_worker.py`, start-up.sh's gunicorn lines, or upgrading gunicorn (the worker overrides gthread internals: `enqueue_req`, `accept`, `on_client_socket_readable`, `murder_keepalived`, `handle`, `handle_request`).
|
||||
- **WAF body inspection**: `python3 scripts/test-waf-body-inspection.py` - offline (runs in the image build); pins `option http-buffer-request`, the `X-WHP-WAF-Body` complete/partial stamp (strip -> stamp -> SPOE -> strip), `tune.bufsize` against the 100 ms SPOE fail-open budget and the 65535-byte SPOP frame cap, and Coraza's body limits in `coraza-spoa/config.yaml`. Run it after touching the Coraza block, `tune.bufsize`, or the SPOE engine.
|
||||
- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired/<stamp>/` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/<primary>` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`.
|
||||
- **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md
|
||||
|
||||
### Reading stick tables (and why it is easy to get silently wrong)
|
||||
|
||||
@@ -45,6 +45,8 @@ COPY ./templates /haproxy/templates
|
||||
COPY requirements.txt /haproxy/
|
||||
COPY haproxy_manager.py /haproxy/
|
||||
COPY haproxy_gunicorn_worker.py /haproxy/
|
||||
# Read by GET /api/capabilities (WHP gates whole-site re-bundling on it).
|
||||
COPY VERSION /haproxy/VERSION
|
||||
COPY scripts /haproxy/scripts
|
||||
COPY trusted_ips.list /etc/haproxy/trusted_ips.list
|
||||
COPY trusted_ips.map /etc/haproxy/trusted_ips.map
|
||||
|
||||
+455
-102
@@ -9,7 +9,7 @@ import psutil
|
||||
import functools
|
||||
import hmac
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import ipaddress
|
||||
import shutil
|
||||
@@ -598,6 +598,19 @@ def backup_existing_pem(dest_path):
|
||||
return None
|
||||
|
||||
|
||||
def leaf_is_expired(path):
|
||||
"""True only when openssl positively says the first certificate in `path`
|
||||
has expired. Unreadable/missing openssl returns False: those cases are
|
||||
rejected, with a better message, by validate_pem_bundle()."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
['openssl', 'x509', '-checkend', '0', '-noout', '-in', path],
|
||||
capture_output=True, text=True, stdin=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return False
|
||||
return proc.returncode == 1 and 'will expire' in (proc.stdout or '')
|
||||
|
||||
|
||||
def publish_pem_bundle(dest_path, source_paths):
|
||||
"""Publish cert+key as a combined PEM at dest_path, atomically.
|
||||
|
||||
@@ -639,6 +652,15 @@ def publish_pem_bundle(dest_path, source_paths):
|
||||
raise CertificatePublishError(
|
||||
f'assembled bundle for {dest_path} is not usable: {msg}')
|
||||
|
||||
# Never publish an expired certificate. HAProxy loads the crt directory
|
||||
# wholesale and serves the first file that claims an SNI name, so an
|
||||
# expired file in there can only ever shadow a valid one (brain-jar.com.pem
|
||||
# did exactly that to brainjar.net on 2026-10-07).
|
||||
if leaf_is_expired(source_paths[0]):
|
||||
raise CertificatePublishError(
|
||||
f'refusing to publish {dest_path}: the certificate in '
|
||||
f'{source_paths[0]} has expired')
|
||||
|
||||
dest_dir = os.path.dirname(dest_path) or '.'
|
||||
try:
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
@@ -788,6 +810,30 @@ def health_check():
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
# What this edge supports. WHP's orchestrator only switches to whole-site
|
||||
# re-bundling with unreachable_names when it sees these (HARD RULE: the WHP side
|
||||
# must not rely on 2026.10.3 behaviour against an older image, whose CN-only
|
||||
# quarantine + `certbot delete` would act on the larger bundles).
|
||||
EDGE_CAPABILITIES = ['bundle_renewed_flag', 'san_overlap_retirement',
|
||||
'unreachable_names', 'retired_lineage_tree']
|
||||
|
||||
|
||||
def _image_version():
|
||||
for path in ('/haproxy/VERSION', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'VERSION')):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return fh.read().strip()
|
||||
except OSError:
|
||||
continue
|
||||
return 'unknown'
|
||||
|
||||
|
||||
@app.route('/api/capabilities', methods=['GET'])
|
||||
@require_api_key
|
||||
def get_capabilities():
|
||||
return jsonify({'version': _image_version(), 'capabilities': EDGE_CAPABILITIES})
|
||||
|
||||
|
||||
@app.route('/api/regenerate', methods=['GET'])
|
||||
@require_api_key
|
||||
def regenerate_conf():
|
||||
@@ -1008,132 +1054,372 @@ def request_ssl():
|
||||
log_operation('request_ssl', False, str(e))
|
||||
return jsonify({'status': 'error', 'message': str(e)}), 500
|
||||
|
||||
def _quarantine_superseded_certs(keep_path, keep_lineage, bundle_names):
|
||||
"""Move aside cert files that the just-issued bundle supersedes.
|
||||
# ---------------------------------------------------------------------------
|
||||
# Superseded-certificate retirement (incident 2026-10-07, brainjar.net)
|
||||
# ---------------------------------------------------------------------------
|
||||
# HAProxy loads /etc/haproxy/certs as a DIRECTORY and, when two files claim the
|
||||
# same SNI name, serves whichever it loaded first. So a new bundle has no effect
|
||||
# while an older file that also lists one of its names is still in there.
|
||||
#
|
||||
# brainjar.net served an EXPIRED certificate for two days for exactly that
|
||||
# reason. The orchestrator re-bundled brainjar.net and its seven siblings
|
||||
# without the dead brain-jar.com, but the cleanup that was meant to move the old
|
||||
# file aside matched on the old certificate's CN only. Its CN was brain-jar.com,
|
||||
# which was not in the new bundle, so brain-jar.com.pem (which also listed all
|
||||
# eight brainjar names) stayed put and kept winning the SNI match. On top of
|
||||
# that, renew-certificates.sh re-published the old lineage from live/ every 12
|
||||
# hours, so moving the file alone would not have stuck.
|
||||
#
|
||||
# What retirement does now:
|
||||
# * any OTHER pem/lineage that shares ANY name with the new bundle (CN or
|
||||
# SAN) is a candidate, not only one whose CN is in the bundle;
|
||||
# * a candidate is moved aside only if no name that this edge still serves
|
||||
# would lose its last valid certificate (the shared-PEM hazard: one bundle
|
||||
# routinely serves several sites' names, see domains_referencing_cert_path);
|
||||
# * nothing is deleted. The pem goes to the cert-retired sibling directory and
|
||||
# the lineage's live/<name>, archive/<name> and renewal/<name>.conf go
|
||||
# together to /etc/letsencrypt/retired/<stamp>/ (retire_lineage_tree), so
|
||||
# certbot stops renewing it, renew-certificates.sh stops re-publishing it,
|
||||
# and the name can still be issued again later. Every move is logged and
|
||||
# can be reversed with mv.
|
||||
# (This used to `certbot delete` the lineage, which is irreversible.)
|
||||
|
||||
A `.pem` in /etc/haproxy/certs/ is "superseded" iff its certificate's CN
|
||||
is one of the bundle's names AND the file isn't the bundle's own combined
|
||||
file. We don't look at SANs of the OLD certs — being the CN is enough,
|
||||
since that's what HAProxy SNI-matches against and what the file
|
||||
convention names it after.
|
||||
# Where certbot keeps its lineages. A module global so the tests can repoint
|
||||
# it; find_certbot_live_dir() predates it and still hardcodes the live path.
|
||||
LETSENCRYPT_DIR = '/etc/letsencrypt'
|
||||
|
||||
This used to os.remove() the file and immediately `certbot delete` the
|
||||
lineage, both BEFORE anything had checked that the replacement bundle was
|
||||
usable — destroying the only two copies of a working certificate on the
|
||||
strength of a `cat` whose exit status nobody read. Now:
|
||||
|
||||
* the superseded file is MOVED to the cert backup directory instead of
|
||||
unlinked, so an operator can put it back by hand;
|
||||
* the certbot lineage is left alone here. Deleting it is irreversible
|
||||
(archive, live and renewal config all go) and recovering means fresh,
|
||||
rate-limited ACME orders, so it happens only in
|
||||
_delete_superseded_lineages(), after HAProxy has actually loaded the
|
||||
replacement.
|
||||
def cert_retired_dir():
|
||||
"""Where retired .pem files are moved. Sibling of the crt dir, never inside."""
|
||||
return _cert_sibling_dir('cert-retired')
|
||||
|
||||
Moving still solves the problem the removal existed for: the old file no
|
||||
longer sits in the crt directory shadowing the new bundle's SNI match.
|
||||
|
||||
Returns a summary dict for logging / API response. Entries in 'removed'
|
||||
carry the lineage name for the deletion phase.
|
||||
def parse_cert_text(text):
|
||||
"""Parse `openssl x509 -noout -subject -nameopt multiline -enddate
|
||||
-fingerprint -sha256 -ext subjectAltName` output.
|
||||
|
||||
Returns {'cn', 'names', 'not_after', 'fingerprint'}; names is the lower-cased
|
||||
set of the CN plus every DNS SAN. not_after is a UTC epoch or None.
|
||||
"""
|
||||
summary = {'removed': [], 'errors': [], 'skipped': []}
|
||||
cn = None
|
||||
names = set()
|
||||
not_after = None
|
||||
fingerprint = None
|
||||
in_san = False
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if line.startswith('commonName') and '=' in line and cn is None:
|
||||
cn = line.split('=', 1)[1].strip().lower()
|
||||
continue
|
||||
if line.startswith('notAfter='):
|
||||
try:
|
||||
not_after = int(datetime.strptime(
|
||||
' '.join(line.split('=', 1)[1].split()),
|
||||
'%b %d %H:%M:%S %Y %Z').replace(
|
||||
tzinfo=timezone.utc).timestamp())
|
||||
except ValueError:
|
||||
not_after = None
|
||||
continue
|
||||
if '=' in line and line.lower().split('=', 1)[0].endswith('fingerprint'):
|
||||
fingerprint = line.split('=', 1)[1].strip().upper()
|
||||
continue
|
||||
if line.startswith('X509v3 Subject Alternative Name'):
|
||||
in_san = True
|
||||
continue
|
||||
if in_san:
|
||||
for part in line.split(','):
|
||||
part = part.strip()
|
||||
if part.startswith('DNS:'):
|
||||
names.add(part[4:].strip().lower())
|
||||
in_san = False
|
||||
if cn:
|
||||
names.add(cn)
|
||||
return {'cn': cn, 'names': names, 'not_after': not_after,
|
||||
'fingerprint': fingerprint}
|
||||
|
||||
|
||||
def read_cert_info(path):
|
||||
"""parse_cert_text() for the first certificate in `path`, or None."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
['openssl', 'x509', '-in', path, '-noout', '-subject',
|
||||
'-nameopt', 'multiline', '-enddate', '-fingerprint', '-sha256',
|
||||
'-ext', 'subjectAltName'],
|
||||
capture_output=True, text=True, stdin=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
info = parse_cert_text(proc.stdout)
|
||||
return info if info['names'] else None
|
||||
|
||||
|
||||
def cert_name_covers(cert_name, host):
|
||||
"""Does certificate name `cert_name` (maybe `*.x`) cover hostname `host`?"""
|
||||
cert_name = (cert_name or '').lower()
|
||||
host = (host or '').lower()
|
||||
if cert_name == host:
|
||||
return True
|
||||
if cert_name.startswith('*.') and not host.startswith('*.'):
|
||||
suffix = cert_name[1:] # ".example.com"
|
||||
head = host[:-len(suffix)] if host.endswith(suffix) else ''
|
||||
return bool(head) and '.' not in head
|
||||
return False
|
||||
|
||||
|
||||
def _cert_covers(names, host):
|
||||
return any(cert_name_covers(n, host) for n in names)
|
||||
|
||||
|
||||
def _served_names():
|
||||
"""Every hostname this edge has a route for (the SQLite domains table)."""
|
||||
try:
|
||||
with sqlite3.connect(DB_FILE) as conn:
|
||||
return {row[0].lower() for row in
|
||||
conn.execute('SELECT domain FROM domains').fetchall()
|
||||
if row[0]}
|
||||
except Exception as e:
|
||||
# Unknown served set -> treat every name as served, i.e. never retire a
|
||||
# still-valid certificate on the strength of a failed query.
|
||||
logger.error("Could not read the domains table for retirement checks: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def retirement_blockers(candidate, others, bundle_names, served_names,
|
||||
unreachable_names, now=None):
|
||||
"""Names that would lose their LAST valid certificate if `candidate` went.
|
||||
|
||||
candidate/others are read_cert_info() dicts. `others` are the certificates
|
||||
that would remain in the crt directory. Pure, so it is unit-testable.
|
||||
|
||||
* An expired candidate blocks nothing: it is not a valid certificate for
|
||||
anyone, and while it sits in the crt dir it can only shadow a good one.
|
||||
* A name the caller asserts is not served here (unreachable_names - DNS
|
||||
points elsewhere or nowhere) blocks nothing.
|
||||
* A name with no route on this edge blocks nothing.
|
||||
* served_names=None (could not read the DB) means "assume everything is
|
||||
served", so a valid candidate is kept unless the bundle covers it all.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
if candidate.get('not_after') is None:
|
||||
return ['<unreadable expiry>']
|
||||
if candidate['not_after'] <= now:
|
||||
return []
|
||||
unreachable = {n.lower() for n in (unreachable_names or ())}
|
||||
bundle = {n.lower() for n in bundle_names}
|
||||
if served_names is None:
|
||||
candidates_to_check = set(candidate['names'])
|
||||
else:
|
||||
candidates_to_check = {s for s in served_names
|
||||
if _cert_covers(candidate['names'], s)}
|
||||
blockers = []
|
||||
for name in sorted(candidates_to_check):
|
||||
if name in unreachable or _cert_covers(bundle, name):
|
||||
continue
|
||||
covered_elsewhere = any(
|
||||
o is not candidate
|
||||
and o.get('fingerprint') != candidate.get('fingerprint')
|
||||
and o.get('not_after') is not None and o['not_after'] > now
|
||||
and _cert_covers(o['names'], name)
|
||||
for o in others)
|
||||
if not covered_elsewhere:
|
||||
blockers.append(name)
|
||||
return blockers
|
||||
|
||||
|
||||
def _overlaps_bundle(info, bundle_names):
|
||||
return any(_cert_covers(info['names'], b) for b in bundle_names) or \
|
||||
any(n in {b.lower() for b in bundle_names} for n in info['names'])
|
||||
|
||||
|
||||
def _unique_dest(directory, name):
|
||||
dest = os.path.join(directory, name)
|
||||
if not os.path.exists(dest):
|
||||
return dest
|
||||
stamp = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
dest = os.path.join(directory, f'{name}.{stamp}')
|
||||
n = 1
|
||||
while os.path.exists(dest):
|
||||
dest = os.path.join(directory, f'{name}.{stamp}.{n}')
|
||||
n += 1
|
||||
return dest
|
||||
|
||||
|
||||
def _quarantine_superseded_certs(keep_path, keep_lineage, bundle_names,
|
||||
unreachable_names=()):
|
||||
"""Move aside .pem files in the crt dir that overlap the new bundle.
|
||||
|
||||
Runs after the new bundle is published and before HAProxy reloads, so the
|
||||
replacement is never the only thing we have. See the block comment above
|
||||
for the rules. Returns {'removed', 'kept', 'skipped', 'errors'}.
|
||||
"""
|
||||
summary = {'removed': [], 'kept': [], 'skipped': [], 'errors': []}
|
||||
if not os.path.isdir(SSL_CERTS_DIR):
|
||||
return summary
|
||||
|
||||
keep_basename = os.path.basename(keep_path)
|
||||
served = _served_names()
|
||||
|
||||
# Inventory once; `remaining` shrinks as files are retired so the coverage
|
||||
# check for the next candidate sees what will actually be left.
|
||||
inventory = {}
|
||||
for fname in sorted(os.listdir(SSL_CERTS_DIR)):
|
||||
if not fname.endswith('.pem'):
|
||||
continue
|
||||
info = read_cert_info(os.path.join(SSL_CERTS_DIR, fname))
|
||||
if info is None:
|
||||
summary['skipped'].append({'file': fname, 'reason': 'openssl could not read it'})
|
||||
continue
|
||||
inventory[fname] = info
|
||||
|
||||
remaining = dict(inventory)
|
||||
for fname, info in inventory.items():
|
||||
if fname == keep_basename:
|
||||
continue
|
||||
fpath = os.path.join(SSL_CERTS_DIR, fname)
|
||||
try:
|
||||
cn_proc = subprocess.run(
|
||||
['openssl', 'x509', '-in', fpath, '-noout', '-subject', '-nameopt', 'multiline'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if cn_proc.returncode != 0:
|
||||
summary['skipped'].append({'file': fname, 'reason': 'openssl read failed'})
|
||||
continue
|
||||
# `-nameopt multiline` lays out the subject one RDN per line; CN is
|
||||
# the row matching `commonName`. Robust against unusual subject orderings.
|
||||
cn = None
|
||||
for line in cn_proc.stdout.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith('commonName'):
|
||||
# format: "commonName = example.com"
|
||||
parts = line.split('=', 1)
|
||||
if len(parts) == 2:
|
||||
cn = parts[1].strip()
|
||||
break
|
||||
if not cn:
|
||||
summary['skipped'].append({'file': fname, 'reason': 'no CN found'})
|
||||
continue
|
||||
except Exception as e:
|
||||
summary['skipped'].append({'file': fname, 'reason': f'inspect failed: {e}'})
|
||||
if not _overlaps_bundle(info, bundle_names):
|
||||
continue
|
||||
|
||||
if cn not in bundle_names:
|
||||
continue # not superseded — different domain group
|
||||
|
||||
# This file's CN is now part of our new bundle — supersede it.
|
||||
lineage_name = fname[:-len('.pem')]
|
||||
if lineage_name == keep_lineage:
|
||||
# Defensive: shouldn't happen because of keep_basename check, but
|
||||
# don't accidentally drop the lineage we just wrote.
|
||||
others = [i for f, i in remaining.items() if f != fname]
|
||||
blockers = retirement_blockers(info, others, bundle_names, served,
|
||||
unreachable_names)
|
||||
if blockers:
|
||||
logger.warning(
|
||||
"Not retiring %s although it overlaps the new %s bundle: it is "
|
||||
"still the only valid certificate for %s", fname, keep_lineage,
|
||||
', '.join(blockers))
|
||||
summary['kept'].append({'file': fname, 'blockers': blockers})
|
||||
continue
|
||||
|
||||
try:
|
||||
os.makedirs(cert_backup_dir(), exist_ok=True)
|
||||
quarantine_path = os.path.join(cert_backup_dir(), fname)
|
||||
# Move, not unlink: out of the crt directory (so it stops shadowing
|
||||
# the new bundle) but still on disk for manual recovery.
|
||||
shutil.move(fpath, quarantine_path)
|
||||
os.makedirs(cert_retired_dir(), exist_ok=True)
|
||||
dest = _unique_dest(cert_retired_dir(), fname)
|
||||
shutil.move(os.path.join(SSL_CERTS_DIR, fname), dest)
|
||||
remaining.pop(fname, None)
|
||||
logger.warning(
|
||||
"Retired superseded certificate %s (names: %s; expires %s) -> %s "
|
||||
"because it overlaps the new %s bundle",
|
||||
fname, ', '.join(sorted(info['names'])),
|
||||
datetime.fromtimestamp(info['not_after'], timezone.utc).isoformat()
|
||||
if info['not_after'] else 'unknown', dest, keep_lineage)
|
||||
summary['removed'].append({
|
||||
'file': fname,
|
||||
'cn': cn,
|
||||
'lineage': lineage_name,
|
||||
'moved_to': quarantine_path,
|
||||
'lineage_deleted': False,
|
||||
'cn': info['cn'],
|
||||
'names': sorted(info['names']),
|
||||
'expired': bool(info['not_after'] and info['not_after'] <= time.time()),
|
||||
'moved_to': dest,
|
||||
})
|
||||
except Exception as e:
|
||||
summary['errors'].append({'file': fname, 'error': str(e)})
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _delete_superseded_lineages(summary):
|
||||
"""`certbot delete` the lineages quarantined by _quarantine_superseded_certs().
|
||||
def retire_lineage_tree(lineage, stamp=None):
|
||||
"""Move a certbot lineage COMPLETELY out of certbot's view, without deleting.
|
||||
|
||||
IRREVERSIBLE: certbot removes the lineage's archive, live symlinks and
|
||||
renewal config. If it turns out we needed that certificate, the only way
|
||||
back is a fresh ACME order, which Let's Encrypt rate-limits — an outage
|
||||
measured in hours. So this is gated hardest of anything in this module: it
|
||||
runs only after the replacement bundle has been assembled, validated,
|
||||
published, AND loaded by a HAProxy that reloaded successfully.
|
||||
live/<x>, archive/<x> and renewal/<x>.conf go together to
|
||||
LETSENCRYPT_DIR/retired/<UTCstamp>/{live,archive,renewal}/. All three must
|
||||
move: certbot's storage.RenewableCert.new_lineage() refuses to create a
|
||||
lineage while archive/<x> exists ("archive directory exists for <x>") - and
|
||||
it has already written an empty renewal/<x>.conf by then, which breaks every
|
||||
later --cert-name <x> and every `certbot renew` parse (review H2, verified
|
||||
against certbot 2.11). Moving only live/ and renewal/ (as the first version
|
||||
of this did) therefore made the name permanently un-issuable.
|
||||
|
||||
Mutates `summary` in place. Best-effort: some files have no corresponding
|
||||
lineage (e.g. self-signed dev certs) and a failure here is harmless — a
|
||||
dead lineage merely wastes a renewal attempt on the next 12h cron tick.
|
||||
The retired tree keeps the same depth as /etc/letsencrypt, so the relative
|
||||
live/<x>/*.pem -> ../../archive/<x>/*N.pem symlinks still resolve, and a
|
||||
restore is three mv's back. Returns {'retired_to': dir, 'moved': [...]}.
|
||||
Raises on failure after moving back anything already moved.
|
||||
"""
|
||||
for entry in summary.get('removed', []):
|
||||
lineage_name = entry.get('lineage')
|
||||
if not lineage_name:
|
||||
stamp = stamp or datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
root = os.path.join(LETSENCRYPT_DIR, 'retired', stamp)
|
||||
n = 1
|
||||
while any(os.path.exists(os.path.join(root, part, name)) for part, name in
|
||||
(('live', lineage), ('archive', lineage), ('renewal', f'{lineage}.conf'))):
|
||||
root = os.path.join(LETSENCRYPT_DIR, 'retired', f'{stamp}.{n}')
|
||||
n += 1
|
||||
plan = [(os.path.join(LETSENCRYPT_DIR, part, name), os.path.join(root, part, name))
|
||||
for part, name in (('renewal', f'{lineage}.conf'), ('live', lineage),
|
||||
('archive', lineage))]
|
||||
moved = []
|
||||
try:
|
||||
for src, dst in plan:
|
||||
if os.path.lexists(src):
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
shutil.move(src, dst)
|
||||
moved.append((src, dst))
|
||||
except Exception:
|
||||
for src, dst in reversed(moved):
|
||||
try:
|
||||
shutil.move(dst, src)
|
||||
except Exception as e: # pragma: no cover - logged, nothing else to do
|
||||
logger.critical("Could not restore %s from %s: %s", src, dst, e)
|
||||
raise
|
||||
return {'retired_to': root, 'moved': [d for _, d in moved]}
|
||||
|
||||
|
||||
def _retire_superseded_lineages(keep_lineage, bundle_names, unreachable_names=(),
|
||||
keep_fingerprint=None):
|
||||
"""Move overlapping certbot lineages out of certbot's tree (retire_lineage_tree).
|
||||
|
||||
Same overlap and safety rules as _quarantine_superseded_certs(), evaluated
|
||||
against the pem files that are left in the crt dir. Runs only after HAProxy
|
||||
has loaded the new bundle. keep_lineage is the bundle's --cert-name and is
|
||||
never retired.
|
||||
"""
|
||||
summary = {'retired': [], 'kept': [], 'errors': []}
|
||||
live_root = os.path.join(LETSENCRYPT_DIR, 'live')
|
||||
if not os.path.isdir(live_root):
|
||||
return summary
|
||||
served = _served_names()
|
||||
|
||||
remaining_pems = []
|
||||
if os.path.isdir(SSL_CERTS_DIR):
|
||||
for fname in sorted(os.listdir(SSL_CERTS_DIR)):
|
||||
if fname.endswith('.pem'):
|
||||
info = read_cert_info(os.path.join(SSL_CERTS_DIR, fname))
|
||||
if info is not None:
|
||||
info['_file'] = fname
|
||||
remaining_pems.append(info)
|
||||
|
||||
for lineage in sorted(os.listdir(live_root)):
|
||||
lineage_dir = os.path.join(live_root, lineage)
|
||||
# Hard guard: the bundle's own --cert-name lineage is never a candidate,
|
||||
# whatever any other check says (review H1).
|
||||
if lineage == keep_lineage or not os.path.isdir(lineage_dir):
|
||||
continue
|
||||
cert_file = os.path.join(lineage_dir, 'cert.pem')
|
||||
if not os.path.exists(cert_file):
|
||||
cert_file = os.path.join(lineage_dir, 'fullchain.pem')
|
||||
info = read_cert_info(cert_file)
|
||||
if info is None or not _overlaps_bundle(info, bundle_names):
|
||||
continue
|
||||
if keep_fingerprint and info.get('fingerprint') == keep_fingerprint:
|
||||
continue
|
||||
# The lineage's own published pem is not "another" certificate, even if
|
||||
# its fingerprint drifted from the lineage (renewed, not republished).
|
||||
others = [p for p in remaining_pems if p['_file'] != f'{lineage}.pem']
|
||||
blockers = retirement_blockers(info, others, bundle_names, served,
|
||||
unreachable_names)
|
||||
if blockers:
|
||||
logger.warning(
|
||||
"Not retiring certbot lineage %s although it overlaps the new %s "
|
||||
"bundle: it is still the only valid certificate for %s",
|
||||
lineage, keep_lineage, ', '.join(blockers))
|
||||
summary['kept'].append({'lineage': lineage, 'blockers': blockers})
|
||||
continue
|
||||
entry = {'lineage': lineage, 'names': sorted(info['names'])}
|
||||
try:
|
||||
cb_proc = subprocess.run(
|
||||
['certbot', 'delete', '--cert-name', lineage_name, '-n'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
entry['lineage_deleted'] = (cb_proc.returncode == 0)
|
||||
if cb_proc.returncode != 0:
|
||||
entry['certbot_stderr'] = (cb_proc.stderr or '').strip()[:200]
|
||||
entry.update(retire_lineage_tree(lineage))
|
||||
logger.warning(
|
||||
"Retired certbot lineage %s (names: %s) because it overlaps the "
|
||||
"new %s bundle: live, archive and renewal conf moved to %s. "
|
||||
"Restore with three mv's back.", lineage,
|
||||
', '.join(entry['names']), keep_lineage, entry['retired_to'])
|
||||
summary['retired'].append(entry)
|
||||
except Exception as e:
|
||||
entry['certbot_error'] = str(e)
|
||||
entry['error'] = str(e)
|
||||
summary['errors'].append(entry)
|
||||
return summary
|
||||
|
||||
|
||||
@app.route('/api/ssl/bundle', methods=['POST'])
|
||||
@require_api_key
|
||||
def request_ssl_bundle():
|
||||
@@ -1144,7 +1430,18 @@ def request_ssl_bundle():
|
||||
/api/ssl calls when a site has multiple domains.
|
||||
|
||||
Body:
|
||||
{"primary": "example.com", "sans": ["www.example.com", ...]}
|
||||
{"primary": "example.com", "sans": ["www.example.com", ...],
|
||||
"unreachable_names": ["dead.example.com", ...]} # optional
|
||||
|
||||
unreachable_names: names the caller (WHP's orchestrator) has established do
|
||||
not resolve to this host. They are left out of the bundle by the caller;
|
||||
here they only tell superseded-certificate retirement that an older cert's
|
||||
coverage of them is not worth keeping (see retirement_blockers()).
|
||||
|
||||
The response says whether certbot actually issued ("renewed") or found the
|
||||
existing lineage not yet due and did nothing, and carries the published
|
||||
certificate's not_after, so the caller can schedule from the real expiry
|
||||
instead of recording a no-op as a renewal.
|
||||
|
||||
The cert lineage uses --cert-name <primary>, so renewal under the same
|
||||
name doesn't proliferate -0001/-0002 dirs (the issue we hit with the
|
||||
@@ -1182,6 +1479,13 @@ def request_ssl_bundle():
|
||||
return jsonify({'status': 'error', 'message': f'invalid SAN: {s!r}'}), 400
|
||||
cleaned_sans.append(s)
|
||||
|
||||
unreachable_names = data.get('unreachable_names') or []
|
||||
if not isinstance(unreachable_names, list) or not all(
|
||||
isinstance(n, str) for n in unreachable_names):
|
||||
return jsonify({'status': 'error',
|
||||
'message': '"unreachable_names" must be a list of strings'}), 400
|
||||
unreachable_names = [n.strip().lower() for n in unreachable_names if n.strip()]
|
||||
|
||||
seen = {primary}
|
||||
names = [primary]
|
||||
for s in cleaned_sans:
|
||||
@@ -1196,8 +1500,11 @@ def request_ssl_bundle():
|
||||
'message': f'Too many SANs ({len(names)}); Let\'s Encrypt limit is 100',
|
||||
}), 400
|
||||
|
||||
# --keep-until-expiring is what -n does anyway when the lineage exists with
|
||||
# the same names; it is spelled out so the no-op case is deliberate, and it
|
||||
# is detected below rather than reported as an issuance.
|
||||
cmd = [
|
||||
'certbot', 'certonly', '-n', '--standalone',
|
||||
'certbot', 'certonly', '-n', '--standalone', '--keep-until-expiring',
|
||||
'--preferred-challenges', 'http', '--http-01-port=8688',
|
||||
'--cert-name', primary,
|
||||
]
|
||||
@@ -1208,6 +1515,17 @@ def request_ssl_bundle():
|
||||
# Defensive: clear any stale lock left by a SIGKILLed prior run.
|
||||
clear_stale_certbot_locks()
|
||||
|
||||
# The lineage is EXACTLY live/<primary>: certbot was given
|
||||
# --cert-name <primary>, which is authoritative. find_certbot_live_dir()
|
||||
# must NOT be used here - it returns the newest of <primary> and
|
||||
# <primary>-NNNN, so after a "not yet due" no-op a stale, more recently
|
||||
# renewed <primary>-0001 would be published as <primary>.pem and the
|
||||
# real <primary> lineage would then be retired as "superseded" (review
|
||||
# H1, reproduced with brainjar.net). fullchain.pem starts with the leaf.
|
||||
lineage_dir = os.path.join(LETSENCRYPT_DIR, 'live', primary)
|
||||
before_info = read_cert_info(os.path.join(lineage_dir, 'fullchain.pem')) \
|
||||
if os.path.isdir(lineage_dir) else None
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
stderr_excerpt = (result.stderr or '').strip()[:800]
|
||||
@@ -1220,9 +1538,8 @@ def request_ssl_bundle():
|
||||
'attempted_names': names,
|
||||
}), 500
|
||||
|
||||
# Locate the lineage. With --cert-name primary, this should be a
|
||||
# stable directory name (no -NNNN suffix on the first issuance).
|
||||
live_dir = find_certbot_live_dir(primary)
|
||||
# --cert-name <primary> means the lineage is live/<primary>, exactly.
|
||||
live_dir = lineage_dir if os.path.isdir(lineage_dir) else None
|
||||
if not live_dir:
|
||||
error_msg = f'Bundle issued but live dir not found for {primary}'
|
||||
log_operation('request_ssl_bundle', False, error_msg)
|
||||
@@ -1232,6 +1549,20 @@ def request_ssl_bundle():
|
||||
key_path = os.path.join(live_dir, 'privkey.pem')
|
||||
combined_path = f'{SSL_CERTS_DIR}/{primary}.pem'
|
||||
|
||||
# Did certbot issue, or did it find the lineage not yet due? Measured by
|
||||
# the certificate itself (fingerprint before vs after), with certbot's
|
||||
# own wording as a second signal.
|
||||
new_info = read_cert_info(cert_path)
|
||||
output = (result.stdout or '') + (result.stderr or '')
|
||||
certbot_not_due = ('not yet due for renewal' in output.lower()
|
||||
or 'no action taken' in output.lower())
|
||||
if before_info is None:
|
||||
renewed = new_info is not None
|
||||
elif new_info is None:
|
||||
renewed = not certbot_not_due
|
||||
else:
|
||||
renewed = new_info.get('fingerprint') != before_info.get('fingerprint')
|
||||
|
||||
os.makedirs(SSL_CERTS_DIR, exist_ok=True)
|
||||
try:
|
||||
publish_pem_bundle(combined_path, [cert_path, key_path])
|
||||
@@ -1273,6 +1604,7 @@ def request_ssl_bundle():
|
||||
keep_path=combined_path,
|
||||
keep_lineage=primary,
|
||||
bundle_names=set(names),
|
||||
unreachable_names=unreachable_names,
|
||||
)
|
||||
|
||||
# Raises if the config does not validate or HAProxy does not reload;
|
||||
@@ -1285,26 +1617,47 @@ def request_ssl_bundle():
|
||||
logger.critical(
|
||||
"HAProxy did not reload after publishing the bundle for "
|
||||
"%s. The superseded certificate files were moved to %s and "
|
||||
"their certbot lineages were NOT deleted, so they can be "
|
||||
"their certbot lineages were NOT retired, so they can be "
|
||||
"restored by hand: %s",
|
||||
primary, cert_backup_dir(),
|
||||
primary, cert_retired_dir(),
|
||||
[e['file'] for e in cleanup_summary['removed']])
|
||||
raise
|
||||
|
||||
_delete_superseded_lineages(cleanup_summary)
|
||||
# Only now that HAProxy has loaded the new bundle: stop certbot renewing
|
||||
# (and renew-certificates.sh re-publishing) overlapping old lineages.
|
||||
lineage_summary = _retire_superseded_lineages(
|
||||
keep_lineage=primary,
|
||||
bundle_names=set(names),
|
||||
unreachable_names=unreachable_names,
|
||||
keep_fingerprint=new_info.get('fingerprint') if new_info else None,
|
||||
)
|
||||
cleanup_summary['lineages'] = lineage_summary
|
||||
|
||||
not_after = new_info.get('not_after') if new_info else None
|
||||
log_operation(
|
||||
'request_ssl_bundle', True,
|
||||
f'SSL bundle issued for {primary} covering {len(names)} names; '
|
||||
f'cleaned up {len(cleanup_summary["removed"])} superseded lineage(s)'
|
||||
f'SSL bundle {"issued" if renewed else "not yet due (existing certificate kept)"} '
|
||||
f'for {primary} covering {len(names)} names; retired '
|
||||
f'{len(cleanup_summary["removed"])} superseded pem(s) and '
|
||||
f'{len(lineage_summary["retired"])} lineage(s)'
|
||||
)
|
||||
return jsonify({
|
||||
'status': 'success',
|
||||
'primary': primary,
|
||||
'names': names,
|
||||
'cert_path': combined_path,
|
||||
'renewed': bool(renewed),
|
||||
'capabilities': EDGE_CAPABILITIES,
|
||||
'certbot_not_due': bool(certbot_not_due),
|
||||
'not_after': not_after,
|
||||
'not_after_iso': datetime.fromtimestamp(not_after, timezone.utc).isoformat()
|
||||
if not_after else None,
|
||||
'fingerprint': new_info.get('fingerprint') if new_info else None,
|
||||
'cleanup': cleanup_summary,
|
||||
'message': f'Bundled certificate obtained for {len(names)} names',
|
||||
'message': (f'Bundled certificate obtained for {len(names)} names'
|
||||
if renewed else
|
||||
f'Existing certificate for {len(names)} names is not yet '
|
||||
'due for renewal; nothing was issued'),
|
||||
})
|
||||
except Exception as e:
|
||||
log_operation('request_ssl_bundle', False, str(e))
|
||||
|
||||
@@ -203,6 +203,82 @@ cert_bundle_valid() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# cert_is_expired CERT_FILE
|
||||
#
|
||||
# Returns 0 only when openssl positively reports that the first certificate in
|
||||
# CERT_FILE has expired. Anything else (not expired, unreadable, no openssl)
|
||||
# returns 1; unreadable material is rejected with a better message by
|
||||
# cert_bundle_valid().
|
||||
#
|
||||
# Why: HAProxy loads the crt directory wholesale and serves the FIRST file that
|
||||
# claims an SNI name. An expired bundle in there is never useful and can only
|
||||
# shadow a valid one - on 2026-10-07 renew-certificates.sh kept re-publishing
|
||||
# the expired brain-jar.com lineage every 12 hours, and HAProxy served it for
|
||||
# brainjar.net instead of the valid brainjar.net bundle next to it.
|
||||
cert_is_expired() {
|
||||
local file="$1" out
|
||||
[ -f "$file" ] || return 1
|
||||
command -v openssl >/dev/null 2>&1 || return 1
|
||||
out="$(openssl x509 -checkend 0 -noout -in "$file" 2>/dev/null </dev/null)"
|
||||
[ $? -eq 1 ] && [[ "$out" == *"will expire"* ]]
|
||||
}
|
||||
|
||||
# cert_not_after CERT_FILE - the notAfter date string, or empty.
|
||||
cert_not_after() {
|
||||
openssl x509 -enddate -noout -in "$1" 2>/dev/null </dev/null | sed 's/^notAfter=//'
|
||||
}
|
||||
|
||||
cert_retired_dir() {
|
||||
if [ -n "${CERT_RETIRED_DIR:-}" ]; then
|
||||
echo "$CERT_RETIRED_DIR"
|
||||
else
|
||||
echo "$(dirname "${SSL_CERTS_DIR:-/etc/haproxy/certs}")/cert-retired"
|
||||
fi
|
||||
}
|
||||
|
||||
# cert_retire_expired_pems
|
||||
#
|
||||
# Move every EXPIRED *.pem out of $SSL_CERTS_DIR into cert-retired/ (a sibling,
|
||||
# never inside the crt dir). Sets CERT_RETIRED_EXPIRED to the number moved.
|
||||
#
|
||||
# Not publishing expired lineages stops the re-publish loop, but a pem that was
|
||||
# published while valid and has expired since stays in the crt dir and keeps
|
||||
# shadowing - that was exactly brain-jar.com.pem (review M2). An expired
|
||||
# certificate is never a valid certificate for anyone, so it can always go: the
|
||||
# only-valid-cert safety rule cannot be violated by moving it. Two exceptions:
|
||||
# * default_self_signed_cert.pem - HAProxy's fallback, regenerated separately;
|
||||
# * the LAST .pem in the directory - a crt directory with no certificate at
|
||||
# all fails the whole :443 bind, which is worse than serving an expired one.
|
||||
# Nothing is deleted; reverse with mv.
|
||||
cert_retire_expired_pems() {
|
||||
local dir="${SSL_CERTS_DIR:-/etc/haproxy/certs}" retired f base dest remaining
|
||||
CERT_RETIRED_EXPIRED=0
|
||||
retired="$(cert_retired_dir)"
|
||||
for f in "$dir"/*.pem; do
|
||||
[ -f "$f" ] || continue
|
||||
base="$(basename "$f")"
|
||||
[ "$base" = "default_self_signed_cert.pem" ] && continue
|
||||
cert_is_expired "$f" || continue
|
||||
remaining="$(find "$dir" -maxdepth 1 -type f -name '*.pem' | wc -l)"
|
||||
if [ "$remaining" -le 1 ]; then
|
||||
log_error "Expired certificate $f is the only certificate left in $dir - NOT retiring it (an empty crt directory fails the :443 bind)"
|
||||
continue
|
||||
fi
|
||||
if ! mkdir -p "$retired"; then
|
||||
log_error "cannot create $retired - expired $f left in place"
|
||||
continue
|
||||
fi
|
||||
dest="$retired/$base"
|
||||
[ -e "$dest" ] && dest="$dest.$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
if mv -- "$f" "$dest"; then
|
||||
log_error "Retired EXPIRED certificate $base (notAfter $(cert_not_after "$dest")) from $dir to $dest - it could only shadow valid certificates"
|
||||
CERT_RETIRED_EXPIRED=$((CERT_RETIRED_EXPIRED + 1))
|
||||
else
|
||||
log_error "failed to move expired $f to $dest"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# cert_publish CERT_FILE KEY_FILE DEST_FILE
|
||||
#
|
||||
# Assemble CERT_FILE + KEY_FILE into DEST_FILE without ever exposing a
|
||||
@@ -227,6 +303,11 @@ cert_publish() {
|
||||
log_error "cert_publish: private key $key_file is missing or empty"
|
||||
return 1
|
||||
fi
|
||||
# (a2) never publish an expired certificate (see cert_is_expired).
|
||||
if cert_is_expired "$cert_file"; then
|
||||
log_error "cert_publish: certificate $cert_file expired ($(cert_not_after "$cert_file")) - refusing to publish $dest_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# (b) assemble in the staging dir - NOT in the certs dir, which HAProxy
|
||||
# scans wholesale.
|
||||
|
||||
@@ -65,13 +65,22 @@ fi
|
||||
# Copy certificates for each domain
|
||||
UPDATED=0
|
||||
FAILED=0
|
||||
SKIPPED_EXPIRED=0
|
||||
|
||||
while read -r domain; do
|
||||
CERT_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/fullchain.pem"
|
||||
KEY_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/privkey.pem"
|
||||
COMBINED_FILE="${SSL_CERTS_DIR}/${domain}.pem"
|
||||
|
||||
if [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
|
||||
if [ -f "$CERT_FILE" ] && cert_is_expired "$CERT_FILE"; then
|
||||
# Never publish an expired lineage. It cannot help any client and,
|
||||
# because HAProxy serves the first file claiming an SNI name, it can
|
||||
# shadow a valid bundle for a name it shares (brainjar.net, 2026-10-07).
|
||||
# Counted separately from FAILED: a dead lineage that keeps expiring is
|
||||
# reported on every run (ERROR log) without making every run "fail".
|
||||
log_error "Skipping EXPIRED certificate lineage $domain (notAfter $(cert_not_after "$CERT_FILE")) - not publishing it"
|
||||
SKIPPED_EXPIRED=$((SKIPPED_EXPIRED + 1))
|
||||
elif [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
|
||||
# Assemble in a staging dir and rename into place. NEVER redirect into
|
||||
# $COMBINED_FILE: the shell truncates the live pem before cat runs, and
|
||||
# HAProxy loads $SSL_CERTS_DIR as a directory, so one bad file there
|
||||
@@ -89,10 +98,15 @@ while read -r domain; do
|
||||
fi
|
||||
done <<< "$DOMAINS"
|
||||
|
||||
log_info "Certificate update completed: $UPDATED updated, $FAILED failed"
|
||||
# Already-published pems that have expired since are moved out too (they
|
||||
# can only shadow; see cert_retire_expired_pems).
|
||||
cert_retire_expired_pems
|
||||
RETIRED_EXPIRED=$CERT_RETIRED_EXPIRED
|
||||
|
||||
log_info "Certificate update completed: $UPDATED updated, $FAILED failed, $SKIPPED_EXPIRED expired lineage(s) skipped, $RETIRED_EXPIRED expired pem(s) retired"
|
||||
|
||||
# Reload HAProxy if any certificates were updated
|
||||
if [ $UPDATED -gt 0 ]; then
|
||||
if [ $UPDATED -gt 0 ] || [ "$RETIRED_EXPIRED" -gt 0 ]; then
|
||||
# Never reload onto unvalidated material: a reload that fails to load the
|
||||
# certs directory drops HTTPS for every site on this host.
|
||||
if ! haproxy_config_ok; then
|
||||
|
||||
@@ -47,13 +47,22 @@ fi
|
||||
# Copy certificates for each domain
|
||||
UPDATED=0
|
||||
FAILED=0
|
||||
SKIPPED_EXPIRED=0
|
||||
|
||||
while read -r domain; do
|
||||
CERT_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/fullchain.pem"
|
||||
KEY_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/privkey.pem"
|
||||
COMBINED_FILE="${SSL_CERTS_DIR}/${domain}.pem"
|
||||
|
||||
if [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
|
||||
if [ -f "$CERT_FILE" ] && cert_is_expired "$CERT_FILE"; then
|
||||
# Never publish an expired lineage. It cannot help any client and,
|
||||
# because HAProxy serves the first file claiming an SNI name, it can
|
||||
# shadow a valid bundle for a name it shares (brainjar.net, 2026-10-07).
|
||||
# Counted separately from FAILED: a dead lineage that keeps expiring is
|
||||
# reported on every run (ERROR log) without making every run "fail".
|
||||
log_error "Skipping EXPIRED certificate lineage $domain (notAfter $(cert_not_after "$CERT_FILE")) - not publishing it"
|
||||
SKIPPED_EXPIRED=$((SKIPPED_EXPIRED + 1))
|
||||
elif [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
|
||||
# Assemble in a staging dir and rename into place. NEVER redirect into
|
||||
# $COMBINED_FILE: the shell truncates the live pem before cat runs, and
|
||||
# HAProxy loads $SSL_CERTS_DIR as a directory, so one bad file there
|
||||
@@ -71,10 +80,15 @@ while read -r domain; do
|
||||
fi
|
||||
done <<< "$DOMAINS"
|
||||
|
||||
log_info "Certificate sync completed: $UPDATED updated, $FAILED failed"
|
||||
# Already-published pems that have expired since are moved out too (they
|
||||
# can only shadow; see cert_retire_expired_pems).
|
||||
cert_retire_expired_pems
|
||||
RETIRED_EXPIRED=$CERT_RETIRED_EXPIRED
|
||||
|
||||
log_info "Certificate sync completed: $UPDATED updated, $FAILED failed, $SKIPPED_EXPIRED expired lineage(s) skipped, $RETIRED_EXPIRED expired pem(s) retired"
|
||||
|
||||
# Reload HAProxy if any certificates were updated
|
||||
if [ $UPDATED -gt 0 ]; then
|
||||
if [ $UPDATED -gt 0 ] || [ "$RETIRED_EXPIRED" -gt 0 ]; then
|
||||
# Never reload onto unvalidated material: a reload that fails to load the
|
||||
# certs directory drops HTTPS for every site on this host.
|
||||
if ! haproxy_config_ok; then
|
||||
|
||||
@@ -73,6 +73,15 @@ LIB = os.path.join(SCRIPTS_DIR, 'cert-publish-lib.sh')
|
||||
BROKEN_TOKEN = '__BROKEN__'
|
||||
DOMAIN = 'test.example.com'
|
||||
|
||||
# Real SAN certificates (see scripts/test-fixtures/san-certs/). expired_old is
|
||||
# the brain-jar.com shape: a multi-SAN certificate that expired in 2020.
|
||||
SAN_FIXTURES = os.path.join(SCRIPTS_DIR, 'test-fixtures', 'san-certs')
|
||||
|
||||
|
||||
def san_fixture(name):
|
||||
with open(os.path.join(SAN_FIXTURES, name)) as fh:
|
||||
return fh.read()
|
||||
|
||||
# --- test key material -------------------------------------------------------
|
||||
# openssl req -x509 -newkey rsa:2048 -keyout key1 -out cert1 -days 3650 -nodes \
|
||||
# -subj /CN=test.example.com
|
||||
@@ -618,6 +627,72 @@ class CertScriptBehaviour:
|
||||
self.assertFalse(self.reload_attempted())
|
||||
|
||||
|
||||
# -- never publish an expired certificate (brainjar.net, 2026-10-07) ----
|
||||
def test_expired_lineage_is_skipped_and_reported(self):
|
||||
"""renew-certificates.sh used to re-publish EVERY live/ lineage every 12h,
|
||||
expired or not; the expired brain-jar.com bundle then shadowed the valid
|
||||
brainjar.net bundle for the names they shared."""
|
||||
dead = os.path.join(self.le_live, 'brain-jar.com')
|
||||
os.makedirs(dead)
|
||||
write(os.path.join(dead, 'fullchain.pem'), san_fixture('expired_old.crt'))
|
||||
write(os.path.join(dead, 'privkey.pem'), san_fixture('expired_old.key'))
|
||||
|
||||
result = self.run_it()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertFalse(
|
||||
os.path.exists(os.path.join(self.certs_dir, 'brain-jar.com.pem')),
|
||||
'an expired lineage was published into the crt directory')
|
||||
# The healthy lineage next to it is still published.
|
||||
self.assertEqual(read(self.live_pem), NEW_BUNDLE)
|
||||
self.assertIn('Skipping EXPIRED certificate lineage brain-jar.com', self.logs())
|
||||
self.assertIn('1 updated, 0 failed, 1 expired lineage(s) skipped', self.logs())
|
||||
self.assertIn('brain-jar.com', read(self.error_log),
|
||||
'the skip must land in the monitored error log')
|
||||
|
||||
def test_expired_lineage_does_not_replace_an_existing_pem(self):
|
||||
dead = os.path.join(self.le_live, 'brain-jar.com')
|
||||
os.makedirs(dead)
|
||||
write(os.path.join(dead, 'fullchain.pem'), san_fixture('expired_old.crt'))
|
||||
write(os.path.join(dead, 'privkey.pem'), san_fixture('expired_old.key'))
|
||||
existing = write(os.path.join(self.certs_dir, 'brain-jar.com.pem'), OTHER_BUNDLE)
|
||||
|
||||
self.run_it()
|
||||
|
||||
self.assertEqual(read(existing), OTHER_BUNDLE,
|
||||
'an expired lineage overwrote the file in the crt dir')
|
||||
|
||||
def test_already_published_expired_pem_is_retired(self):
|
||||
"""Review M2: skipping publication is not enough - an expired pem that
|
||||
is ALREADY in the crt dir (brain-jar.com.pem) keeps shadowing."""
|
||||
expired = write(os.path.join(self.certs_dir, 'brain-jar.com.pem'),
|
||||
san_fixture('expired_old.crt') + san_fixture('expired_old.key'))
|
||||
result = self.run_it()
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertFalse(os.path.exists(expired), 'expired pem still in the crt dir')
|
||||
retired = os.path.join(self.haproxy_dir, 'cert-retired', 'brain-jar.com.pem')
|
||||
self.assertTrue(os.path.exists(retired), 'expired pem was not moved to cert-retired/ (never delete)')
|
||||
self.assertIn('Retired EXPIRED certificate brain-jar.com.pem', self.logs())
|
||||
self.assertIn('1 expired pem(s) retired', self.logs())
|
||||
self.assertTrue(self.reload_attempted())
|
||||
self.assertEqual(read(self.live_pem), NEW_BUNDLE, 'valid neighbours untouched')
|
||||
self.assert_certs_dir_is_clean()
|
||||
|
||||
def test_retiring_an_expired_pem_alone_triggers_a_reload(self):
|
||||
self.seed_previous_bundle(NEW_BUNDLE)
|
||||
shutil.rmtree(self.domain_dir)
|
||||
other = os.path.join(self.le_live, 'other.example')
|
||||
os.makedirs(other)
|
||||
write(os.path.join(other, 'fullchain.pem'), san_fixture('expired_old.crt'))
|
||||
write(os.path.join(other, 'privkey.pem'), san_fixture('expired_old.key'))
|
||||
write(os.path.join(self.certs_dir, 'other.example.pem'),
|
||||
san_fixture('expired_old.crt') + san_fixture('expired_old.key'))
|
||||
result = self.run_it()
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertIn('0 updated', self.logs())
|
||||
self.assertTrue(self.reload_attempted(), 'HAProxy must reload to drop the retired pem')
|
||||
|
||||
|
||||
class TestRenewCertificates(CertScriptBehaviour, CertScriptFixture):
|
||||
SCRIPT = 'renew-certificates.sh'
|
||||
|
||||
@@ -638,6 +713,52 @@ class TestCertPublishLibrary(CertScriptFixture):
|
||||
path = write(os.path.join(self.tmp, 'ok.pem'), NEW_BUNDLE)
|
||||
self.assertEqual(self.call('cert_bundle_valid "$1"', path).returncode, 0)
|
||||
|
||||
def test_cert_is_expired_reports_both_directions(self):
|
||||
expired = write(os.path.join(self.tmp, 'expired.crt'), san_fixture('expired_old.crt'))
|
||||
valid = write(os.path.join(self.tmp, 'valid.crt'), san_fixture('new_bundle.crt'))
|
||||
self.assertEqual(0, self.call('cert_is_expired "$1"', expired).returncode,
|
||||
'an expired certificate was not recognised as expired')
|
||||
self.assertEqual(1, self.call('cert_is_expired "$1"', valid).returncode,
|
||||
'a valid certificate was reported as expired')
|
||||
self.assertEqual(1, self.call('cert_is_expired "$1"',
|
||||
os.path.join(self.tmp, 'missing')).returncode)
|
||||
|
||||
def test_expired_sweep_keeps_the_last_pem_and_the_default_cert(self):
|
||||
exp = san_fixture('expired_old.crt') + san_fixture('expired_old.key')
|
||||
only = write(os.path.join(self.certs_dir, 'only.pem'), exp)
|
||||
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
|
||||
self.assertIn('n=0', r.stdout)
|
||||
self.assertTrue(os.path.exists(only), 'emptied the crt dir: that fails the whole :443 bind')
|
||||
dflt = write(os.path.join(self.certs_dir, 'default_self_signed_cert.pem'), exp)
|
||||
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
|
||||
self.assertIn('n=1', r.stdout)
|
||||
self.assertTrue(os.path.exists(dflt), 'the default self-signed cert is not ours to retire')
|
||||
self.assertFalse(os.path.exists(only))
|
||||
|
||||
def test_expired_sweep_never_touches_a_valid_pem(self):
|
||||
valid = write(os.path.join(self.certs_dir, 'a.pem'), NEW_BUNDLE)
|
||||
write(os.path.join(self.certs_dir, 'b.pem'), OTHER_BUNDLE)
|
||||
r = self.call('cert_retire_expired_pems; echo "n=$CERT_RETIRED_EXPIRED"')
|
||||
self.assertIn('n=0', r.stdout)
|
||||
self.assertEqual(read(valid), NEW_BUNDLE)
|
||||
|
||||
def test_cert_publish_refuses_an_expired_certificate(self):
|
||||
cert = write(os.path.join(self.tmp, 'expired.crt'), san_fixture('expired_old.crt'))
|
||||
key = write(os.path.join(self.tmp, 'expired.key'), san_fixture('expired_old.key'))
|
||||
dest = os.path.join(self.certs_dir, 'brain-jar.com.pem')
|
||||
|
||||
result = self.call('cert_publish "$1" "$2" "$3"', cert, key, dest)
|
||||
|
||||
self.assert_rejected(result, because='expired')
|
||||
self.assertFalse(os.path.exists(dest), 'expired certificate was published')
|
||||
# And the same material with a valid certificate IS publishable, so the
|
||||
# refusal above is about expiry, not about the fixture.
|
||||
good_cert = write(os.path.join(self.tmp, 'good.crt'), san_fixture('new_bundle.crt'))
|
||||
good_key = write(os.path.join(self.tmp, 'good.key'), san_fixture('new_bundle.key'))
|
||||
ok = self.call('cert_publish "$1" "$2" "$3"', good_cert, good_key, dest)
|
||||
self.assertEqual(0, ok.returncode, ok.stdout + ok.stderr)
|
||||
self.assertTrue(os.path.exists(dest))
|
||||
|
||||
def test_empty_and_missing_bundles_rejected(self):
|
||||
empty = write(os.path.join(self.tmp, 'empty.pem'), '')
|
||||
self.assert_rejected(self.call('cert_bundle_valid "$1"', empty),
|
||||
|
||||
@@ -57,6 +57,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import unittest
|
||||
|
||||
BROKEN_TOKEN = '__BROKEN__'
|
||||
@@ -254,6 +255,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
|
||||
# certbot stub: succeeds, announces a renewal, and records every invocation so
|
||||
# tests can assert that `certbot delete` did or did not run.
|
||||
# TEST_CERTBOT_INSTALL_SRC/_DIR: on certonly, copy fullchain.pem + privkey.pem
|
||||
# from SRC into DIR (a real issuance changes the lineage's certificate).
|
||||
# TEST_CERTBOT_NOT_DUE: on certonly, print certbot's "not yet due" wording.
|
||||
FAKE_CERTBOT = textwrap.dedent("""\
|
||||
#!/bin/sh
|
||||
if [ -n "$TEST_CERTBOT_LOG" ]; then
|
||||
@@ -262,6 +266,14 @@ FAKE_CERTBOT = textwrap.dedent("""\
|
||||
case "$1" in
|
||||
renew) echo "Congratulations, all renewals succeeded" ;;
|
||||
delete) [ -n "$TEST_CERTBOT_DELETE_FAILS" ] && exit 1 ;;
|
||||
certonly)
|
||||
if [ -n "$TEST_CERTBOT_INSTALL_SRC" ]; then
|
||||
mkdir -p "$TEST_CERTBOT_INSTALL_DIR"
|
||||
cp "$TEST_CERTBOT_INSTALL_SRC/fullchain.pem" "$TEST_CERTBOT_INSTALL_SRC/privkey.pem" "$TEST_CERTBOT_INSTALL_DIR/"
|
||||
fi
|
||||
if [ -n "$TEST_CERTBOT_NOT_DUE" ]; then
|
||||
echo "Certificate not yet due for renewal; no action taken."
|
||||
fi ;;
|
||||
esac
|
||||
exit 0
|
||||
""")
|
||||
@@ -331,6 +343,7 @@ class CertPublishTestCase(unittest.TestCase):
|
||||
'CORAZA_SPOE_BACKUP_PATH': os.path.join(self.etc, 'coraza-spoe.cfg.backup'),
|
||||
'CLUSTER_SECRET_PATH': os.path.join(self.etc, 'cluster-secret'),
|
||||
'SSL_CERTS_DIR': self.certs,
|
||||
'LETSENCRYPT_DIR': os.path.join(self.tmp, 'letsencrypt'),
|
||||
'HAPROXY_SOCKET_PATH': os.path.join(self.etc, 'haproxy.sock'),
|
||||
'API_KEY': None,
|
||||
}
|
||||
@@ -574,7 +587,7 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
|
||||
|
||||
@FIX_ONLY
|
||||
@NEEDS_OPENSSL
|
||||
def test_lineage_is_deleted_only_after_haproxy_loads_the_bundle(self):
|
||||
def test_lineage_is_untouched_when_haproxy_does_not_load_the_bundle(self):
|
||||
"""A reload failure must leave the lineage intact and the file recoverable."""
|
||||
old_path = self._setup_supersede(broken=False)
|
||||
# Make generate_config() produce a config the validator rejects, so the
|
||||
@@ -591,7 +604,7 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
|
||||
[], self.certbot_deletes(),
|
||||
'the lineage must not be deleted when HAProxy did not reload')
|
||||
self.assertFalse(os.path.exists(old_path))
|
||||
quarantined = os.path.join(hm.cert_backup_dir(),
|
||||
quarantined = os.path.join(hm.cert_retired_dir(),
|
||||
os.path.basename(old_path))
|
||||
self.assertTrue(os.path.exists(quarantined),
|
||||
'the superseded file must be recoverable by hand')
|
||||
@@ -599,8 +612,9 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
|
||||
|
||||
@FIX_ONLY
|
||||
@NEEDS_OPENSSL
|
||||
def test_successful_bundle_still_supersedes_and_deletes(self):
|
||||
"""The cleanup must still do its job on the happy path."""
|
||||
def test_successful_bundle_still_supersedes_without_deleting(self):
|
||||
"""The cleanup must still do its job on the happy path - by moving, never
|
||||
by `certbot delete` (irreversible; recovery is a rate-limited ACME order)."""
|
||||
old_path = self._setup_supersede(broken=False)
|
||||
|
||||
resp = self.client.post('/api/ssl/bundle', json={
|
||||
@@ -612,12 +626,381 @@ class TestOldCertificateIsNotDestroyedFirst(CertPublishTestCase):
|
||||
self.assertFalse(os.path.exists(old_path),
|
||||
'the superseded file must leave the crt directory or '
|
||||
'it keeps shadowing the new bundle')
|
||||
self.assertEqual(['delete --cert-name www.test.example.com -n'],
|
||||
self.certbot_deletes())
|
||||
self.assertTrue(os.path.exists(os.path.join(
|
||||
hm.cert_retired_dir(), os.path.basename(old_path))),
|
||||
'the superseded file must be moved aside, not destroyed')
|
||||
self.assertEqual([], self.certbot_deletes(),
|
||||
'retirement must never `certbot delete` anything')
|
||||
self.assertTrue(self.edge_would_start())
|
||||
self.assert_only_final_pems_in_certs_dir()
|
||||
|
||||
|
||||
|
||||
# --- SAN-overlap retirement (brainjar.net, 2026-10-07) -----------------------
|
||||
SAN_FIXTURES = os.path.join(MODULE_DIR, 'scripts', 'test-fixtures', 'san-certs')
|
||||
|
||||
|
||||
def san(name):
|
||||
with open(os.path.join(SAN_FIXTURES, name)) as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def make_certbot_lineage(le_dir, lineage, cert, key):
|
||||
"""archive/<x>/{cert,chain,fullchain,privkey}1.pem + live/<x>/*.pem
|
||||
symlinks (../../archive/<x>/...1.pem) + renewal/<x>.conf, like certbot."""
|
||||
archive = os.path.join(le_dir, 'archive', lineage)
|
||||
live = os.path.join(le_dir, 'live', lineage)
|
||||
os.makedirs(archive, exist_ok=True)
|
||||
os.makedirs(live, exist_ok=True)
|
||||
for kind, body in (('cert', cert), ('chain', cert), ('fullchain', cert), ('privkey', key)):
|
||||
with open(os.path.join(archive, f'{kind}1.pem'), 'w') as fh:
|
||||
fh.write(body)
|
||||
link = os.path.join(live, f'{kind}.pem')
|
||||
if os.path.lexists(link):
|
||||
os.remove(link)
|
||||
os.symlink(f'../../archive/{lineage}/{kind}1.pem', link)
|
||||
os.makedirs(os.path.join(le_dir, 'renewal'), exist_ok=True)
|
||||
with open(os.path.join(le_dir, 'renewal', f'{lineage}.conf'), 'w') as fh:
|
||||
fh.write(f'archive_dir = {archive}\ncert = {live}/cert.pem\n')
|
||||
return live
|
||||
|
||||
|
||||
def retired_path(le_dir, part, name):
|
||||
hits = sorted(__import__('glob').glob(os.path.join(le_dir, 'retired', '*', part, name)))
|
||||
return hits[-1] if hits else os.path.join(le_dir, 'retired', '<none>', part, name)
|
||||
|
||||
|
||||
FIX_ONLY_RETIRE = unittest.skipIf(
|
||||
TESTING_FOREIGN_TREE and not hasattr(hm, 'retirement_blockers'),
|
||||
'HAPROXY_MANAGER_DIR points at a tree without SAN-overlap retirement')
|
||||
|
||||
|
||||
class TestRetirementRules(unittest.TestCase):
|
||||
"""retirement_blockers() is the shared-PEM safety rule, tested directly."""
|
||||
|
||||
NOW = 2_000_000_000
|
||||
|
||||
def cert(self, names, days_left, fp):
|
||||
return {'names': set(names), 'fingerprint': fp,
|
||||
'not_after': self.NOW + days_left * 86400, 'cn': sorted(names)[0]}
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_expired_candidate_never_blocks(self):
|
||||
old = self.cert(['brain-jar.com', 'brainjar.net'], -2, 'A')
|
||||
self.assertEqual([], hm.retirement_blockers(
|
||||
old, [], {'brainjar.net'}, {'brain-jar.com', 'brainjar.net'}, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_valid_candidate_that_is_last_cert_for_a_served_name_is_kept(self):
|
||||
old = self.cert(['shared.example', 'brainjar.net', 'other.example'], 40, 'A')
|
||||
self.assertEqual(['other.example'], hm.retirement_blockers(
|
||||
old, [], {'brainjar.net'}, {'brainjar.net', 'other.example'}, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_unreachable_names_do_not_block(self):
|
||||
old = self.cert(['dead.example', 'brainjar.net'], 40, 'A')
|
||||
self.assertEqual([], hm.retirement_blockers(
|
||||
old, [], {'brainjar.net'}, {'brainjar.net', 'dead.example'},
|
||||
['dead.example'], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_name_covered_by_another_valid_cert_does_not_block(self):
|
||||
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
|
||||
other = self.cert(['other.example'], 60, 'B')
|
||||
self.assertEqual([], hm.retirement_blockers(
|
||||
old, [other], {'brainjar.net'}, {'other.example'}, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_cover_by_an_expired_or_identical_cert_does_not_count(self):
|
||||
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
|
||||
expired = self.cert(['other.example'], -1, 'B')
|
||||
same = self.cert(['other.example'], 60, 'A')
|
||||
self.assertEqual(['other.example'], hm.retirement_blockers(
|
||||
old, [expired, same], {'brainjar.net'}, {'other.example'}, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_unknown_served_set_keeps_every_valid_name(self):
|
||||
old = self.cert(['brainjar.net', 'other.example'], 40, 'A')
|
||||
self.assertEqual(['other.example'], hm.retirement_blockers(
|
||||
old, [], {'brainjar.net'}, None, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_wildcard_coverage(self):
|
||||
self.assertTrue(hm.cert_name_covers('*.example.com', 'a.example.com'))
|
||||
self.assertFalse(hm.cert_name_covers('*.example.com', 'a.b.example.com'))
|
||||
self.assertFalse(hm.cert_name_covers('*.example.com', 'example.com'))
|
||||
old = self.cert(['*.example.com', 'brainjar.net'], 40, 'A')
|
||||
self.assertEqual(['shop.example.com'], hm.retirement_blockers(
|
||||
old, [], {'brainjar.net'}, {'shop.example.com'}, [], now=self.NOW))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_read_cert_info_reads_sans_and_expiry(self):
|
||||
tmp = tempfile.mkdtemp(prefix='cert-info-')
|
||||
self.addCleanup(shutil.rmtree, tmp, True)
|
||||
path = os.path.join(tmp, 'c.pem')
|
||||
with open(path, 'w') as fh:
|
||||
fh.write(san('expired_old.crt'))
|
||||
info = hm.read_cert_info(path)
|
||||
self.assertEqual({'brain-jar.com', 'www.brain-jar.com', 'brainjar.net',
|
||||
'www.brainjar.net'}, info['names'])
|
||||
self.assertEqual('brain-jar.com', info['cn'])
|
||||
self.assertEqual(1585699200, info['not_after']) # 2020-04-01T00:00:00Z
|
||||
self.assertTrue(info['fingerprint'])
|
||||
|
||||
|
||||
class TestSanOverlapRetirement(CertPublishTestCase):
|
||||
"""The new bundle must not stay shadowed by an older cert sharing a SAN."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.le = os.path.join(self.tmp, 'letsencrypt')
|
||||
self.new_src = os.path.join(self.tmp, 'issued', 'brainjar.net')
|
||||
self.write(os.path.join(self.new_src, 'fullchain.pem'), san('new_bundle.crt'))
|
||||
self.write(os.path.join(self.new_src, 'privkey.pem'), san('new_bundle.key'))
|
||||
# certbot "issues" brainjar.net into live/ when called.
|
||||
os.environ['TEST_CERTBOT_INSTALL_SRC'] = self.new_src
|
||||
os.environ['TEST_CERTBOT_INSTALL_DIR'] = os.path.join(self.le_live, 'brainjar.net')
|
||||
for name in ('brainjar.net', 'www.brainjar.net'):
|
||||
self.add_domain(name, name.replace('.', '_'))
|
||||
|
||||
def old_lineage(self, lineage, fixture, publish=True):
|
||||
"""A lineage laid out the way certbot does it: real files in
|
||||
archive/<x>/*1.pem, relative symlinks in live/<x>/, a renewal conf."""
|
||||
live = make_certbot_lineage(self.le, lineage, san(f'{fixture}.crt'),
|
||||
san(f'{fixture}.key'))
|
||||
conf = os.path.join(self.le, 'renewal', f'{lineage}.conf')
|
||||
if not publish:
|
||||
return live, conf, None
|
||||
pem = self.write(os.path.join(self.certs, f'{lineage}.pem'),
|
||||
san(f'{fixture}.crt') + san(f'{fixture}.key'))
|
||||
return live, conf, pem
|
||||
|
||||
def bundle(self, **extra):
|
||||
body = {'primary': 'brainjar.net', 'sans': ['www.brainjar.net']}
|
||||
body.update(extra)
|
||||
resp = self.client.post('/api/ssl/bundle', json=body)
|
||||
self.assertEqual(200, resp.status_code, resp.get_data(as_text=True))
|
||||
return resp.get_json()
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_expired_overlapping_cert_with_a_different_cn_is_retired(self):
|
||||
"""The incident: CN brain-jar.com, SANs include brainjar.net."""
|
||||
self.add_domain('brain-jar.com', 'bjc')
|
||||
self.add_domain('www.brain-jar.com', 'wbjc')
|
||||
live, conf, pem = self.old_lineage('brain-jar.com', 'expired_old')
|
||||
|
||||
data = self.bundle(unreachable_names=['brain-jar.com', 'www.brain-jar.com'])
|
||||
|
||||
self.assertFalse(os.path.exists(pem),
|
||||
'the expired SAN-overlapping pem is still shadowing the bundle')
|
||||
retired = [e['file'] for e in data['cleanup']['removed']]
|
||||
self.assertEqual(['brain-jar.com.pem'], retired)
|
||||
self.assertTrue(os.path.exists(os.path.join(hm.cert_retired_dir(), 'brain-jar.com.pem')))
|
||||
# Lineage retired, not deleted: renew-certificates.sh can no longer
|
||||
# re-publish it and certbot can no longer try to renew it.
|
||||
self.assertFalse(os.path.exists(live))
|
||||
self.assertFalse(os.path.exists(conf))
|
||||
# live, archive AND renewal leave together (review H2), into one tree.
|
||||
self.assertFalse(os.path.exists(os.path.join(self.le, 'archive', 'brain-jar.com')))
|
||||
self.assertTrue(os.path.isdir(retired_path(self.le, 'live', 'brain-jar.com')))
|
||||
self.assertTrue(os.path.isdir(retired_path(self.le, 'archive', 'brain-jar.com')))
|
||||
self.assertTrue(os.path.exists(retired_path(self.le, 'renewal', 'brain-jar.com.conf')))
|
||||
# Relative symlinks still resolve in the retired tree (restore = mv back).
|
||||
self.assertIn('BEGIN CERTIFICATE', self.read(
|
||||
os.path.join(retired_path(self.le, 'live', 'brain-jar.com'), 'cert.pem')))
|
||||
self.assertEqual([], self.certbot_deletes())
|
||||
self.assertTrue(os.path.exists(os.path.join(self.certs, 'brainjar.net.pem')))
|
||||
self.assertTrue(self.edge_would_start())
|
||||
self.assert_only_final_pems_in_certs_dir()
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_expired_overlap_is_retired_even_without_the_unreachable_hint(self):
|
||||
self.add_domain('brain-jar.com', 'bjc')
|
||||
live, conf, pem = self.old_lineage('brain-jar.com', 'expired_old')
|
||||
self.bundle()
|
||||
self.assertFalse(os.path.exists(pem))
|
||||
self.assertFalse(os.path.exists(live))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_shared_pem_still_serving_another_site_is_kept(self):
|
||||
"""The shared-PEM hazard: never retire the only valid cert for a served name."""
|
||||
self.add_domain('other-site.example', 'other_site')
|
||||
live, conf, pem = self.old_lineage('shared-old.example', 'shared_old')
|
||||
|
||||
data = self.bundle()
|
||||
|
||||
self.assertTrue(os.path.exists(pem),
|
||||
'retired the only valid certificate for other-site.example')
|
||||
self.assertTrue(os.path.exists(live))
|
||||
self.assertTrue(os.path.exists(conf))
|
||||
kept = {e['file']: e['blockers'] for e in data['cleanup']['kept']}
|
||||
self.assertEqual({'shared-old.example.pem': ['other-site.example']}, kept)
|
||||
self.assertEqual(['other-site.example'],
|
||||
data['cleanup']['lineages']['kept'][0]['blockers'])
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_valid_overlap_whose_other_names_are_unreachable_is_retired(self):
|
||||
self.add_domain('dead.example', 'dead')
|
||||
live, conf, pem = self.old_lineage('dead.example', 'dead_old')
|
||||
|
||||
self.bundle(unreachable_names=['dead.example'])
|
||||
|
||||
self.assertFalse(os.path.exists(pem))
|
||||
self.assertFalse(os.path.exists(live))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_valid_overlap_still_served_without_the_hint_is_kept(self):
|
||||
"""Same files as above, but nobody said dead.example is gone: keep it."""
|
||||
self.add_domain('dead.example', 'dead')
|
||||
live, conf, pem = self.old_lineage('dead.example', 'dead_old')
|
||||
|
||||
self.bundle()
|
||||
|
||||
self.assertTrue(os.path.exists(pem))
|
||||
self.assertTrue(os.path.exists(live))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_non_overlapping_certificates_are_untouched(self):
|
||||
live, conf, pem = self.old_lineage('unrelated.example', 'unrelated')
|
||||
self.bundle()
|
||||
self.assertTrue(os.path.exists(pem), 'retired a cert sharing no name with the bundle')
|
||||
self.assertTrue(os.path.exists(live))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_response_distinguishes_issuance_from_not_yet_due(self):
|
||||
issued = self.bundle()
|
||||
self.assertTrue(issued['renewed'], 'a fresh issuance was not reported as one')
|
||||
self.assertEqual(4922899200, issued['not_after']) # 2126-01-01T00:00:00Z
|
||||
|
||||
# Lineage now exists with the same names; certbot keeps it.
|
||||
del os.environ['TEST_CERTBOT_INSTALL_SRC']
|
||||
os.environ['TEST_CERTBOT_NOT_DUE'] = '1'
|
||||
noop = self.bundle()
|
||||
self.assertFalse(noop['renewed'], 'a certbot no-op was reported as a renewal')
|
||||
self.assertTrue(noop['certbot_not_due'])
|
||||
self.assertEqual(4922899200, noop['not_after'])
|
||||
self.assertIn('--keep-until-expiring', self.certbot_invocations()[-1])
|
||||
|
||||
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_not_due_noop_never_retires_its_own_cert_name_lineage(self):
|
||||
"""Review H1: find_certbot_live_dir() returns the NEWEST of brainjar.net
|
||||
and brainjar.net-0001. After a certbot no-op that was the stale -0001:
|
||||
it got published as brainjar.net.pem and the real brainjar.net lineage
|
||||
was retired as "superseded"."""
|
||||
del os.environ['TEST_CERTBOT_INSTALL_SRC']
|
||||
os.environ['TEST_CERTBOT_NOT_DUE'] = '1'
|
||||
real = make_certbot_lineage(self.le, 'brainjar.net', san('new_bundle.crt'), san('new_bundle.key'))
|
||||
stale = make_certbot_lineage(self.le, 'brainjar.net-0001', san('dead_old.crt'), san('dead_old.key'))
|
||||
later = time.time() + 60
|
||||
os.utime(os.path.join(self.le, 'archive', 'brainjar.net-0001', 'fullchain1.pem'), (later, later))
|
||||
# Production behaviour of find_certbot_live_dir(): newest wins.
|
||||
hm.find_certbot_live_dir = lambda base: stale if base == 'brainjar.net' else None
|
||||
|
||||
data = self.bundle()
|
||||
|
||||
self.assertTrue(os.path.isdir(real), 'the --cert-name lineage itself was retired')
|
||||
self.assertTrue(os.path.exists(os.path.join(self.le, 'renewal', 'brainjar.net.conf')))
|
||||
self.assertNotIn('brainjar.net', [e['lineage'] for e in data['cleanup']['lineages']['retired']])
|
||||
published = self.read(os.path.join(self.certs, 'brainjar.net.pem'))
|
||||
self.assertIn(san('new_bundle.crt').strip().splitlines()[1], published,
|
||||
'published the stale -0001 certificate instead of live/brainjar.net')
|
||||
self.assertFalse(data['renewed'])
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_bundle_reports_capabilities(self):
|
||||
data = self.bundle()
|
||||
self.assertIn('san_overlap_retirement', data.get('capabilities', []))
|
||||
caps = self.client.get('/api/capabilities').get_json()
|
||||
self.assertIn('unreachable_names', caps['capabilities'])
|
||||
|
||||
|
||||
class TestRetiredLineageCanBeReissued(CertPublishTestCase):
|
||||
"""Review H2: retirement must leave certbot able to issue the name again."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.le = hm.LETSENCRYPT_DIR
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_retirement_removes_every_part_certbot_checks(self):
|
||||
make_certbot_lineage(self.le, 'brain-jar.com', san('expired_old.crt'), san('expired_old.key'))
|
||||
out = hm.retire_lineage_tree('brain-jar.com')
|
||||
# certbot new_lineage() refuses if any of these exist for the name.
|
||||
for part in (('live', 'brain-jar.com'), ('archive', 'brain-jar.com'),
|
||||
('renewal', 'brain-jar.com.conf')):
|
||||
self.assertFalse(os.path.lexists(os.path.join(self.le, *part)), part)
|
||||
self.assertTrue(os.path.lexists(os.path.join(out['retired_to'], *part)), part)
|
||||
self.assertIn('BEGIN CERTIFICATE',
|
||||
self.read(os.path.join(out['retired_to'], 'live', 'brain-jar.com', 'fullchain.pem')))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_second_retirement_of_the_same_name_does_not_collide(self):
|
||||
make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key'))
|
||||
a = hm.retire_lineage_tree('x.example', stamp='S')
|
||||
make_certbot_lineage(self.le, 'x.example', san('new_bundle.crt'), san('new_bundle.key'))
|
||||
b = hm.retire_lineage_tree('x.example', stamp='S')
|
||||
self.assertNotEqual(a['retired_to'], b['retired_to'])
|
||||
self.assertTrue(os.path.isdir(os.path.join(a['retired_to'], 'archive', 'x.example')))
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
def test_certbot_can_issue_the_name_again_after_retirement(self):
|
||||
"""Against certbot's own storage code (skipped where certbot is not
|
||||
importable, e.g. the image's app python)."""
|
||||
# Import AND initialisation are both guarded: a half-installed or
|
||||
# version-skewed certbot can fail with far more than ImportError (a
|
||||
# missing transitive dependency, a pkg_resources/entry-point error, a
|
||||
# moved internal). Any of those means "cannot run this check here",
|
||||
# not "retirement is broken". new_lineage() itself is NOT inside the
|
||||
# guard - that is the assertion.
|
||||
try:
|
||||
import types
|
||||
from certbot._internal import storage
|
||||
real_rv = storage.relevant_values
|
||||
new_lineage = storage.RenewableCert.new_lineage
|
||||
cfg = types.SimpleNamespace(
|
||||
config_dir=self.le, renewal_configs_dir=os.path.join(self.le, 'renewal'),
|
||||
default_archive_dir=os.path.join(self.le, 'archive'),
|
||||
live_dir=os.path.join(self.le, 'live'))
|
||||
except Exception as e: # noqa: BLE001 - deliberately broad, see above
|
||||
self.skipTest(f'certbot unusable from this python ({type(e).__name__}: {e})')
|
||||
storage.relevant_values = lambda c: {'server': 'https://acme.invalid/directory'}
|
||||
self.addCleanup(setattr, storage, 'relevant_values', real_rv)
|
||||
cert, key = san('new_bundle.crt').encode(), san('new_bundle.key').encode()
|
||||
|
||||
first = new_lineage('brain-jar.com', cert, key, cert, cfg)
|
||||
self.assertEqual('brain-jar.com', first.lineagename)
|
||||
hm.retire_lineage_tree('brain-jar.com')
|
||||
again = new_lineage('brain-jar.com', cert, key, cert, cfg)
|
||||
self.assertEqual('brain-jar.com', again.lineagename,
|
||||
'certbot could not re-create the lineage under the same name')
|
||||
self.assertTrue(os.path.getsize(os.path.join(self.le, 'renewal', 'brain-jar.com.conf')) > 0)
|
||||
|
||||
class TestExpiredCertificateIsNeverPublished(CertPublishTestCase):
|
||||
|
||||
@FIX_ONLY_RETIRE
|
||||
@NEEDS_OPENSSL
|
||||
def test_publisher_refuses_an_expired_leaf(self):
|
||||
src = os.path.join(self.tmp, 'expired')
|
||||
cert = self.write(os.path.join(src, 'fullchain.pem'), san('expired_old.crt'))
|
||||
key = self.write(os.path.join(src, 'privkey.pem'), san('expired_old.key'))
|
||||
live = self.publish_live_bundle('brain-jar.com')
|
||||
with self.assertRaises(hm.CertificatePublishError) as ctx:
|
||||
hm.publish_pem_bundle(live, [cert, key])
|
||||
self.assertIn('expired', str(ctx.exception))
|
||||
self.assertEqual(PREVIOUS_BUNDLE, self.read(live))
|
||||
|
||||
|
||||
class TestBundleValidation(CertPublishTestCase):
|
||||
"""What may and may not be published."""
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIC9DCCAdygAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMZGVh
|
||||
ZC5leGFtcGxlMCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAwWjAXMRUw
|
||||
EwYDVQQDDAxkZWFkLmV4YW1wbGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQCumL3T27a3D6fZbIgwovdH6SfvKAkz3NwBiDLTtIn+Zia3l5FynrGXWZnO
|
||||
abvnQNxEVqyYtTJOn9taitVc30wLVpk31BQdLab5rdVa4IADnd3Mel5Cd0M+P3BU
|
||||
SpW96rK40KVftzzDEUnH+3ncuzPQ7sKXom0JYqBXT42s8LF5UKtXI1oF1iW/7uz0
|
||||
seOIdyP/n3T2cX1wsCa/F0v2MAyze5FhdtWij+2IK1FRbIbSSr6zUCAeAWdqiM9b
|
||||
PBd2cBaBwTe/3TICu0RnvvRBvlMNrgbEBm+wnDf2qlxAO2pSER1bipi4C9gu1nm2
|
||||
wzG4zlKU31fdLvQOzMGH1W5H91sBAgMBAAGjSDBGMCUGA1UdEQQeMByCDGRlYWQu
|
||||
ZXhhbXBsZYIMYnJhaW5qYXIubmV0MB0GA1UdDgQWBBSPDJiv0Z2YpuloexOfvpIQ
|
||||
UrWufTANBgkqhkiG9w0BAQsFAAOCAQEAJwBtN3Qw41XvYuSa8Mjn869QOTKEgkaR
|
||||
EuNQ8WXVBOTRJdxaLHDunGO4ExjHHswCWzXkefYOoIClAU9qLnk9M667BSNxOcz0
|
||||
VcG/DUMd0tO8X4Zm0zOMjt4VC7whgj7MdYtYVg2+Jhb4sMSx6OqmCUAD4X6ULSTn
|
||||
lZK2holEC/H3bKt8Q6cecgeubgWojP1bWmWK7jQL5NRIzxQ0KzFxgAf3t2Z05nX8
|
||||
loABkBiR7vDthkQseUW6RQMayDyzhVqORLt+gZ9D5T3hT1M7VaDc4Twr7UbWPFcR
|
||||
GuQ7RwKplPIIbPe7+c91pm+agdSDjnV5HmKcAMCwf7bWGQ9/Xc0BFg==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCumL3T27a3D6fZ
|
||||
bIgwovdH6SfvKAkz3NwBiDLTtIn+Zia3l5FynrGXWZnOabvnQNxEVqyYtTJOn9ta
|
||||
itVc30wLVpk31BQdLab5rdVa4IADnd3Mel5Cd0M+P3BUSpW96rK40KVftzzDEUnH
|
||||
+3ncuzPQ7sKXom0JYqBXT42s8LF5UKtXI1oF1iW/7uz0seOIdyP/n3T2cX1wsCa/
|
||||
F0v2MAyze5FhdtWij+2IK1FRbIbSSr6zUCAeAWdqiM9bPBd2cBaBwTe/3TICu0Rn
|
||||
vvRBvlMNrgbEBm+wnDf2qlxAO2pSER1bipi4C9gu1nm2wzG4zlKU31fdLvQOzMGH
|
||||
1W5H91sBAgMBAAECggEADUvYBFLn6gPWSZj55nxOqR+2Lmtwh04omgbu2fpWhPTE
|
||||
ujgKd/23srnnMZXnR6qZQQIPGwJiOPnJuRjiu/PMZSj8j54bbwfA6XmbH9SCIzm0
|
||||
la50UGBSkw0T50jBI76vICMm6PBkf0rT1oKN9VOd/qlkJoCUiFcGCh177Fbt07oY
|
||||
R3iAKu7/+yEANX0iEcxbO330TPW3oI0ah5OryfKVoNvxpF8NUfV9pWY66APkOYMJ
|
||||
zsYvIMvzV/KIgfrtyxJjSjgd7Ud8uictYG1BrZBXpIuGxuxyKUJ7OXOvMMVNnEME
|
||||
2uzk/aL75UjpurJJU0LCywdUOGwRqS2q4PYQiTeDLQKBgQDjdywH7RODoYeH05Rd
|
||||
sa2MkNx0Day1i3A4IBds55mzn8jxpT55HlqvHWhNA30kEw01H8+6sB7iMX04xQtS
|
||||
u3N2opR1VlCvMJhGyRUYUtfu4NnAi/HSu7mCOFGAwaLclI7BAI2or2UrclrlUhGb
|
||||
CYIBKfEr6Tn6b6AffWaOruCbtQKBgQDEf70R922fnJK+//QuiYoaESwJd3/k10yJ
|
||||
nH0ZAgbnFcNLT9YVFxHvlt4RHGIotaBjfN+FKMn045jeekgeVdX+uF8+7FwEazAH
|
||||
zOfxswOeKd8EOooqCD7n7cW3Ma6ev8oWmXwJsz2coRW7Z5zCGQWSzCEDB4yFU9Mt
|
||||
0z6aN9SJnQKBgQCvBZSIqL+6KgD53zArA7sRDKnEVn1M13TKis7bij02/DyGyuoO
|
||||
EfAfDBakI4oXxt0Dwm15LWkmDIRYuS0kRH3si96FhpvgoRiaZsDnjV7/PF3wjP3O
|
||||
c4mjDCiSVOw2o8xP71gQspy62GveRc6sJ52zvY0Hh5aH/3oKoNfX8/F5fQKBgALQ
|
||||
vbnDLMvrp2KbMjl0eQkUk+fy0V9LAmq6VkSEderbsfejRjL5KKWhKL9gA7wHliu2
|
||||
v1EeG36pwCf34o6ktjM3Gw5AvrTFgGOsOOqKoROkD/D0kcQ2aAhUrM+KI7FcmB2o
|
||||
x1EwQ0K6QCB2RJW3XBAVYqezdPz4DX+LlksJX781AoGBAISGOer3s4+oRjO/2VuT
|
||||
hnMQrET3A0GTol8cgYL2N9wfx7jy5bhSfiJIX/dkh5IrPGYoBdWM87gcUgNQ8AZ7
|
||||
il5dVeG80oc7KFU/QltZJu9VS32vwXnNsnrXupQ5Ee0ztTLciTMWU5jxvB0ZJHtK
|
||||
StgYepr4IPDUaObqrPx4Jvfm
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -0,0 +1,19 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDGjCCAgKgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwGDEWMBQGA1UEAwwNYnJh
|
||||
aW4tamFyLmNvbTAeFw0yMDAxMDEwMDAwMDBaFw0yMDA0MDEwMDAwMDBaMBgxFjAU
|
||||
BgNVBAMMDWJyYWluLWphci5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQC9SJUMsK5HhVEATu/FWwJsvaK7lkigh7EhlkxTNkLFNzlgkLcoKMhD2F9s
|
||||
i5KzgYp0K2z899wQAKuLzZcde6BUnw6RHzheCRsIqLZlbY8B3Ayd6VhSMYqhWoXJ
|
||||
7jax2QWFMQVhyy6M5QTCHUVJACMpega+KrsYuNkNAxLc2uGhgHLPnnOoDhGXosyN
|
||||
N8eolukhvFxQNU3E7oo1jBzzzSN+FFPCZ8oWab6ydW7YSxZLlxRWoQ+2mL9I+SSy
|
||||
UEc+Gje6dJLZ9CdJtSQSNGnFoqOUGIipNpS2rrBGctVWijM9y459skTFKbamLPy8
|
||||
FHC7atJOe5EoaekaAXw5Tq+yB+ODAgMBAAGjbjBsMEsGA1UdEQREMEKCDWJyYWlu
|
||||
LWphci5jb22CEXd3dy5icmFpbi1qYXIuY29tggxicmFpbmphci5uZXSCEHd3dy5i
|
||||
cmFpbmphci5uZXQwHQYDVR0OBBYEFAWqbBh2PDUvqQzZoyh960cpuhcOMA0GCSqG
|
||||
SIb3DQEBCwUAA4IBAQCaovtrXl56YcFlXWMrW/XZBe2/fXI14860qyiR5rb07lS6
|
||||
CbKOsnRyQdxZ3cdfRwTN1UMmz5IPOZ6fhdo3IPaS2Wae1OdeupRaRWr1SAJtpVZ3
|
||||
WWugDaumsWV5eWVxVTf93yDEkMERWtTGMviCKBeeHjzHIFiD/EjZznE87V5DYwuf
|
||||
guOd1F4wXCMRFu8q7sBdPXjQ8rZMlQT+dQXzMST9Ee5PXiInmCAA5nT8vMInAwd2
|
||||
rP6FxuHJB75ekYzs57CJ82BKKboOCESjUbm54wmVV5pBL305/EjxGf+VovNlRktg
|
||||
t1znolntIIofE25UCbVNPdrtuDkZgKT3Rv8tCdTP
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC9SJUMsK5HhVEA
|
||||
Tu/FWwJsvaK7lkigh7EhlkxTNkLFNzlgkLcoKMhD2F9si5KzgYp0K2z899wQAKuL
|
||||
zZcde6BUnw6RHzheCRsIqLZlbY8B3Ayd6VhSMYqhWoXJ7jax2QWFMQVhyy6M5QTC
|
||||
HUVJACMpega+KrsYuNkNAxLc2uGhgHLPnnOoDhGXosyNN8eolukhvFxQNU3E7oo1
|
||||
jBzzzSN+FFPCZ8oWab6ydW7YSxZLlxRWoQ+2mL9I+SSyUEc+Gje6dJLZ9CdJtSQS
|
||||
NGnFoqOUGIipNpS2rrBGctVWijM9y459skTFKbamLPy8FHC7atJOe5EoaekaAXw5
|
||||
Tq+yB+ODAgMBAAECggEABAeOCL71eYH4jKRMOBbXu98483u1CxnTEa8klnw9ZVES
|
||||
JctJjY96B/EMwv+g/V+gxhgAkiK9VWfg6bEplOVDWgR9fRBoP57mVkOv3QPn5fcR
|
||||
xlt9LQQZ+pULQRZoUAFEkVEkV1sfIBJmDLMufOnbAHCKa9+hVlrlCqu0lJgM8put
|
||||
2bkpSAqwroHVfmD0SmKFAZggBhFwXFJzwmUf3WL+N25f4RZpJsX17FOJ216mlhQ5
|
||||
edje6dcDpmko2VQtGJXYr8Ge02ZYeRyKGcTsgUZlarB4xUZq1pcnUUSy0Ez7LVbk
|
||||
tKjPcUpZLlnAA/a2pDgHmwN1+fv/X+Wwv/vZaQMBTQKBgQDuZFyfDRSHCssHtQ4+
|
||||
Ezxg4WEjwXkq58WxL0zTtLTcquZfQZ1VXgLwAylVwver0I4/UgCfW/AYY+2eLrba
|
||||
FhuYOosd/3rsUQqf9y+TkSkhe1uVS2/1zjzC1G/5coleLddjzd+YBpEtVxfxDRbA
|
||||
rVePY3GxkJiTrjAg5OETk7GYpwKBgQDLQ6bPhaocJPQmDi3hKyW8zPJ8wAJNyI0c
|
||||
RWt9gjeidjmMvwCKtJbBDcV5V0oQ5SB2M5E49mT58B62totAD7FS18c4a4133haV
|
||||
a1pLBX0U524eQgDzMJ6YA6N9bHdXOH4FZd+Pw90bHetbT1HcKpOdXv4SF/XITju5
|
||||
Ro5xiymdxQKBgBB4fq2+iZyK8lwl9kXzrkhDSiLeSzDbspz7V6OI6LYWWLRcJDKx
|
||||
vCI5FbIHNnV4g1t8nu5ju8LzpIc5UwjeXim2YcZA0I1jTAihW3gmn2Dg1QMLo0Co
|
||||
bypG4zWo5DTNlgQfRUOLsJwhWFYfQWMuls4hYMmT4/bQIiMjrSldGvghAoGALpaB
|
||||
7DyZobvCjdpgH6TREHX0+izUGlnmJd2vZmPrNnxccmrwx9neV789TLN6EWdCynTF
|
||||
S0YGHNcxYvVx8yvlm7RDQCY+lYO8r2CcoGdQpPLqZVsBdrvlDbh/x2aN79rN35Ta
|
||||
6cudqPJxmEsU2x6TGNXF7AW72r47qBqI2XNpBXUCgYAFQdHRRxa2E5rgV0WkL7PF
|
||||
vXok3isAzpnKAsb5oxPZbhIdLo5EPUGCslsTE1Na8NBdUSoyjhgkMELEDx/ceGL7
|
||||
IIlNrc/4j71+IahljzaX/D5b5qaW+RnoTVAlbMRR7L/ecbkoovhQR9d37QdoiD/t
|
||||
B9si92+XB4HYDAjXFxTK3w==
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -0,0 +1,18 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIC+DCCAeCgAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMYnJh
|
||||
aW5qYXIubmV0MCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAwWjAXMRUw
|
||||
EwYDVQQDDAxicmFpbmphci5uZXQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQCyhL5YcFhPEekc/xNMaEHO0hjoNRd5yNhG3BF+pbChYJhkjw5McdH89fdb
|
||||
l4twhlkcyvyaEbcHIgLEgz5caANrlGnzQBTOBTILhA0leTF9hBCQX1yKEotQFdgU
|
||||
QcAAvdAyP77jDF6lX11oCjJwzoT0YeLcFG3XyfDQ7hRSJNJFDqvbOgCc9kZ+B3zz
|
||||
XRVp4l2WyqEUJmKxcpppAnfbVhokwy2417CumYp8wIzYJ57mRk6RrRRNbENw3wGU
|
||||
0v78QfI+ueCw261UWZ+iXwlZt/icvzWtcuoPVxid3uPeRPGV40RUbPGuetvIjLNQ
|
||||
dFOfKjH0QGInyByji9eAaXlmvCKtAgMBAAGjTDBKMCkGA1UdEQQiMCCCDGJyYWlu
|
||||
amFyLm5ldIIQd3d3LmJyYWluamFyLm5ldDAdBgNVHQ4EFgQUdDxfZKX96j61mW7O
|
||||
NiQpN4T+sZ4wDQYJKoZIhvcNAQELBQADggEBAJE1lP62caByZT7GkKtJLfOj84Ih
|
||||
MeP751w2gdU8ZfiO8GxXXiR3ZjYzbJgsEi9LiNT9jqHeBf3pv/mJLjPVZ+U557ws
|
||||
Gq3V5A3uKlRKGcejXlHaWnExhWqLyIaLTeLKstbJyiB/ja4yeTL2TYcOvvlu6weh
|
||||
ERqwx0p6r7LSQ3nRfTdjEhNq3FVUmqQ8neBP/vXLp2hKr2ACZndGyQOiv0oZWzud
|
||||
part6RMd8gRNq1Tctrl0YqRYoLRIddBrnTtUtac4FLqxLSBNc03/I9ZgF56Yf+Sl
|
||||
M1ikmyDJ2ZYJFU6v42Og4/8o3fpq0ttP6pKPi8fInGaFQ+KXO1NWFf1ogD0=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCyhL5YcFhPEekc
|
||||
/xNMaEHO0hjoNRd5yNhG3BF+pbChYJhkjw5McdH89fdbl4twhlkcyvyaEbcHIgLE
|
||||
gz5caANrlGnzQBTOBTILhA0leTF9hBCQX1yKEotQFdgUQcAAvdAyP77jDF6lX11o
|
||||
CjJwzoT0YeLcFG3XyfDQ7hRSJNJFDqvbOgCc9kZ+B3zzXRVp4l2WyqEUJmKxcppp
|
||||
AnfbVhokwy2417CumYp8wIzYJ57mRk6RrRRNbENw3wGU0v78QfI+ueCw261UWZ+i
|
||||
XwlZt/icvzWtcuoPVxid3uPeRPGV40RUbPGuetvIjLNQdFOfKjH0QGInyByji9eA
|
||||
aXlmvCKtAgMBAAECggEAEQi8lvjFXGojmco3H7PVD7SZftLOFidVh9Fo8/JAe+2Z
|
||||
LQM/hIjZ1i7sNverbQXMzAQMfXESS2Jx7Clv0mZ0aRgE0hq7wXrabyHfFUPOtqDa
|
||||
6cPHsz8YvwAOmcQ6W3fBPYI6pzSfdV03rv1GGeCC7ETI3mIupwgUbANDuiDcVY1q
|
||||
+VuK/hDzPJ5KslDHUNR42MgWacIftq99fV9jN0zQQw1EcyO8QwpXzTSEbdyiAkT+
|
||||
n/tR9He2MYhgRhjyWPMaxmY6XQpv1h5ASq1yT1kI2oNTaRT3xvd40sLkbfwpZnxi
|
||||
HnqzCkPptEpWeCLrme+r71+Hxu/+Bph7tPtcuK7jgQKBgQD8XCoPRiWbgCl6kM2G
|
||||
E3GeM6fftTieTIs7sFVte7wutIKlsec4vizc94ipGaoT7yGEG0T8g9+5hAMbebc2
|
||||
zzhxvJMB2s18Z/He+Ujv1L1AhBBNVY3zCAxfdq8/n5MGXppEdVRo9QOQndoBfuAV
|
||||
Qw2Vp4bcy728n4T7sBugW0Ad7QKBgQC1F+utuiF8XI2yx0mX0+Fbty6o+odyM0eb
|
||||
G/QQDlcGpY0K7V3WAFMESNfytig8OhidzRfWn8lNoqhd80SJfCuiYjiGJ8tjBzcp
|
||||
+Vqrd/KAxJySJ3NJ2MwyAp7W3l3HAEZLGVYVCankoa4O3qR8YqaIZwc2nk5+8KnG
|
||||
nBYFlSF/wQKBgQCqaRYh0E2O7wGTeXq/ohukJ9+hqf88rhWLTeyyDM8rfsuGVlb8
|
||||
MzeaUzEed20pHt60FbIFDowh+yoaAJwez9mevCdNJlKCQrLKze1Zvn+5tDRJ9I7x
|
||||
YSrxv/Rhc0FWGMX1moFQirtIib9wKRvqX6gpucHREWfjmpizWN2e8F+glQKBgAOp
|
||||
/jy/nVnUzZEmcxi9Urv2Ut1cZE0tOG+s+x0wDRa3Vsk30EQ6MXPOvX/Nq7a8Pi9D
|
||||
LQ5MoVgOyHVOLL7TYlagwiAEwhq41KbmwqzhemickcZrQevmlQsV/ny+dwtFuCZg
|
||||
1tsKHQPhOZJXcuCJZnEwFHdIcLIk0GqSUee/ArSBAoGALOD+Opk+0j4ObNcqtrpP
|
||||
UcEYqhOuAscbxLoUv53wnOvc7XzMG8JCK1y5LthB2Xanr1JSlheMRrvGxC+WlDmc
|
||||
5v0Nz71cq5QQ1p8AXnosLRjuy/+edMvGIxxwFMEJBO2ra2Jizpdwd34bjpQ42+CQ
|
||||
6pVwFQX0oet8SI5xWUWx7NU=
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -0,0 +1,19 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDGjCCAgKgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwHTEbMBkGA1UEAwwSc2hh
|
||||
cmVkLW9sZC5leGFtcGxlMCAXDTI2MDEwMTAwMDAwMFoYDzIxMjYwMTAxMDAwMDAw
|
||||
WjAdMRswGQYDVQQDDBJzaGFyZWQtb2xkLmV4YW1wbGUwggEiMA0GCSqGSIb3DQEB
|
||||
AQUAA4IBDwAwggEKAoIBAQCBg7iYuvjhIG7nlFeja5Lyv2t2fdtNvxzUib7ygejC
|
||||
+WQvNdH4k12q8vCXNYuadbGAceSejG4bazpjo5/ym2KFZPwiSZoqKUugi9lCoKuL
|
||||
2EUmCHoTsr+CLPuZXyjxu/x250m5llJOhgMHYFwURjARRAku3a/GJUHQR0C572mW
|
||||
mt/WtWueHlqEFoI7jCuBAUmww1p1YNV0cl64fGeYSOaYOHEOp/VqIRlzsy7eGQh3
|
||||
xLQzEMyXzrjD54PZ6NuJeT7heZHVO3if2JbqFxvijCLnBz5WsrS/S/VWOELNEfGr
|
||||
84/Jcwze9ZV8e0y4tOwh9qT82adbl1nyAgLfWuSgp5hRAgMBAAGjYjBgMD8GA1Ud
|
||||
EQQ4MDaCEnNoYXJlZC1vbGQuZXhhbXBsZYIMYnJhaW5qYXIubmV0ghJvdGhlci1z
|
||||
aXRlLmV4YW1wbGUwHQYDVR0OBBYEFP+AdZ9MkBmFjAQTgLnWZxbJtnvkMA0GCSqG
|
||||
SIb3DQEBCwUAA4IBAQA32Xl/1IrOyESd2UQcWp/fuHvkivLgamJrheKvKP0Y0vMV
|
||||
atRu0D0GhqMOr79OT60H/H1FFW7bWQ+UPcZpnuXec40/PcrHxwaSoiLoIUVWhB4t
|
||||
BOhtTDReX29gEjnB7OqBCJu4ub3HnpFTflUZd7n5A4EcetBsowc5EZ9UAtfvkgGD
|
||||
nK9ONvNVGurk/V8xgtMJ877ZK1fn+UtgvIWp/F1Gc/taLhe+oCZOPkOqvvqUnS8q
|
||||
QyIhqmMWRUkBP+jwRA4TP4pj7OH38VY/Ae408Xe4Y97GF7NbOjAL0bkX/bHkLnSG
|
||||
WzoGO0TK7pCemeFQ+Ghic8Yl0eDynoP7rZIhCKS0
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCBg7iYuvjhIG7n
|
||||
lFeja5Lyv2t2fdtNvxzUib7ygejC+WQvNdH4k12q8vCXNYuadbGAceSejG4bazpj
|
||||
o5/ym2KFZPwiSZoqKUugi9lCoKuL2EUmCHoTsr+CLPuZXyjxu/x250m5llJOhgMH
|
||||
YFwURjARRAku3a/GJUHQR0C572mWmt/WtWueHlqEFoI7jCuBAUmww1p1YNV0cl64
|
||||
fGeYSOaYOHEOp/VqIRlzsy7eGQh3xLQzEMyXzrjD54PZ6NuJeT7heZHVO3if2Jbq
|
||||
FxvijCLnBz5WsrS/S/VWOELNEfGr84/Jcwze9ZV8e0y4tOwh9qT82adbl1nyAgLf
|
||||
WuSgp5hRAgMBAAECggEACB1tvTwciKvp5wliGF45feDv5c5q+vXE462h0OYXjyfl
|
||||
1sNuMuoLjhYyFvdcQO2rlTBjB8Ald1hXoeidsVYlF9XvqmJRjjS6MAazMpEzTnba
|
||||
66V6U1j5gOQ6pi3SjfBJGFYo8L+cy/NqUIBWz0+6hp/BsjW3uW2aMGer2Y/zs2Oi
|
||||
s0kUNqqUWQpGpEDxbDnUAzto4OOcTOO4A5NB5Oe2ifzFiXCWtTNqFIC0ae9tiqGy
|
||||
m0sggF2hzmlH1WC54rbv7xujYT6tOAEi1OKp/Cpj8XVLbjaqju5avRl1t2nDRQri
|
||||
9vnOI9P4bnJKbIYzwJVyy8B1M5iK+W2K7wUQ4ZMetQKBgQC2zwtywQ4jUduLWf4f
|
||||
6HLUIvmcHdJI0YBHS1+4fKc7tsakRo/0FHxNdUlBhqkYTyp57h6/R1T2gG70Kla1
|
||||
MarGIgyPIlUXsRojlUcNyKXqSVs+kejftIClb+VDXWrpKtGtyxuESsMjaJBsBBAN
|
||||
hg3yUx2NL2ct4nnlpl1iq3pH9wKBgQC1XkrjntuzjVYlFeg2ih5va3PUD2qmH9Ej
|
||||
leNXTGhzZL1eQWz4GaRCgzJH8q41pXBxcXQ8hMXZCWdW5GHBYfVqjTRKMUPdI6FI
|
||||
ypwMbbTBfQfw1kz5VHfLH3romxikNQXeorZkIZqv7/aLPsxLY78XdW8nSDIsT9ZP
|
||||
mh7It+Tf9wKBgHT7VSYf6D4t2cH42qfshd6+auC+LJywg9gboatbCWEq+z6QcPoO
|
||||
SeT/Utt9oqONtW7FNCPPL812LopMtaPJnrCfhU3q+GzqxgFTO17UsWeVfbAyisTs
|
||||
qw4y5pEPLKT846TFPlyPwwPbm9+jUep1iGMzUMvSM8KnFL4ikcvcU6nNAoGBAJQI
|
||||
MVHV0HadzokYd19EnpDMgE6+goqy/21vIo6XY82DlPjAp+NccnCDSt/kmQJaVQuS
|
||||
0Y+vwxhhj4IhiBRiEwBz3hEMYtq1ASg7juCqDlW66mdNUqhSpDAbMqN2K8ejK3ZL
|
||||
EllqACrHzlBa4j5KXKcWB7JlP+ZhB4gTQDsNtXaDAoGBAJCktIdyM3b6UtM5OofB
|
||||
lzn6/2uT5u46ECBRybk/J+Y2gfcS2cFzkH179SUB4WgfR0h21FlLmjns5ZvHVk0p
|
||||
/U4aY3/J42JU6lCYrsyV5BBH+O0stSmrUG/4JdKRe6Mq/pB321RsWD06LVJrBDpi
|
||||
6NUPW03tNLc+Qe+cnQu0c7y+
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -0,0 +1,18 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIC8zCCAdugAwIBAgICEAQwDQYJKoZIhvcNAQELBQAwHDEaMBgGA1UEAwwRdW5y
|
||||
ZWxhdGVkLmV4YW1wbGUwHhcNMjAwMTAxMDAwMDAwWhcNMjAwNDAxMDAwMDAwWjAc
|
||||
MRowGAYDVQQDDBF1bnJlbGF0ZWQuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQAD
|
||||
ggEPADCCAQoCggEBALEpfqdRPBSrbzixAxD2WhsO1AOTUd/Gguf3zjDZ3TUmWJKb
|
||||
OhlFthtU6PJVhMdAqOk+hiz9Coo/UngWjykyNMOj1YIpq3M4/P8wBTNxkJd/z2x0
|
||||
iAQBKvWlB6hxf9c1kenu/ea1iMIThA/uudtX3g8k39Asj/37+o070cdBJExWGnbG
|
||||
lz103FCO6kBzlJPGIVQwzKH6OKthTzi7Uxy5Fxc/ymS9gc43F0tyY6892Wyl8Y62
|
||||
a3r1PpZoWR0GDFxPIyCS7NP44T15DSL/2RBi4YnWXOC8QMvPnHbSKjoQ6GLQtGkO
|
||||
1NO4Uy3XbwDFanYXv6Wxv0oUmZMuY7CRTOMdxCMCAwEAAaM/MD0wHAYDVR0RBBUw
|
||||
E4IRdW5yZWxhdGVkLmV4YW1wbGUwHQYDVR0OBBYEFBh0L/5U5f94TylibBCgS9pD
|
||||
/VXkMA0GCSqGSIb3DQEBCwUAA4IBAQB5MLqNjTqIdyztjJsKPdYUkA27NpfdqqY6
|
||||
Yprv5N7U0slsGUqFIDrSuJ1mMMwTpt02IAFyCOWMUtze+/0PqM9k9TqxJH2cTODp
|
||||
0ET34Qh8NTo0B534EQUWhny+EzgNAzWK02wbpgm88+Mp/t//sHzhWzOm7uk1dlkq
|
||||
+5ty/8OveWwAtqAWlDtcsRhkEjzynZKUCAF1extwUSuapFSm20mJ4ivt7Ff5MkTI
|
||||
fq8P+3XAxWTaBW8DzBIzYd1mwFHK2pcklLMRRTW3hukPxEcTivBgJdwsHMskM3UO
|
||||
/ejy5EHem5ohCMbwMybvUqLS4xdoqXV36Sj2I1zv0IjgO0wgdK1r
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCxKX6nUTwUq284
|
||||
sQMQ9lobDtQDk1HfxoLn984w2d01JliSmzoZRbYbVOjyVYTHQKjpPoYs/QqKP1J4
|
||||
Fo8pMjTDo9WCKatzOPz/MAUzcZCXf89sdIgEASr1pQeocX/XNZHp7v3mtYjCE4QP
|
||||
7rnbV94PJN/QLI/9+/qNO9HHQSRMVhp2xpc9dNxQjupAc5STxiFUMMyh+jirYU84
|
||||
u1McuRcXP8pkvYHONxdLcmOvPdlspfGOtmt69T6WaFkdBgxcTyMgkuzT+OE9eQ0i
|
||||
/9kQYuGJ1lzgvEDLz5x20io6EOhi0LRpDtTTuFMt128AxWp2F7+lsb9KFJmTLmOw
|
||||
kUzjHcQjAgMBAAECggEAJjBKo+iH6fC2beMCRTLFY5XDr+2etX0GFNi+8prT6zQT
|
||||
3a5mTLJROf0Ipcw3bfgLMRQQOjxlfD2AGnDKdVfkNdPoH0G6OoErUBra1CcDwxSC
|
||||
8t7/vtU9nGvxhabJputeQ64HvyacjzFDw2dzWsvLCzqooCgq9dXOQYmGg8wP9gAx
|
||||
pxfKnd84h41C3pXERdufi6nJHPjZPRW+dKsQ1NMLmtgO5YQnTt3LfVOkZaRa7Izq
|
||||
ZFWZCBfdsNHHuXCYj3FtqYrqSxELuZAxJ62hxpizrfQdKixguGTObHhC4muIo9fR
|
||||
eIMCvrAzrs4Fs8nM/K/IPyDU9wPwINK6Z+sxJkhM+QKBgQDaL3Yt3T/LE6bTYyb0
|
||||
EjVcXoybnuAjFQAenCTIsBRJv21Yv2ddqjq6rOTD64hFuFWPMEOi1TJxIPsFFDMw
|
||||
NeAwF41fiSrtU8awXs0hTKRQeHjElSBckumOl1DrHu2oKuwWXLniViD62vMsWmpZ
|
||||
89KT2rgFQuMPvsD3pps8l6W2HwKBgQDP3eTwdVJTdRYlZonqd0+LHG7h715TT1OE
|
||||
QJOdN9vK5g/9GOCOGHoUxB3IxJVmvz57PrfWaD2rSZ6JMs4TcAY5a0PbJEpgqoJY
|
||||
kIm/EDnSrFxu2L8DNsAzuThfzXrwWc7PzajYJctYtEfvxCyfo/GdZahg2xaEU0Le
|
||||
dp+7fxtJfQKBgGVdLWtLYarH+3Yzl1JG71F1f5+A5SKHTCHsas5tt083xuY3Lh9+
|
||||
yYAWeMVotEunwuAeDR4mLCQQrW4Dn3BjZz8QghaDVFUj2afwsx4oTCIyhfDWU02O
|
||||
10DunaJOIR1Cq6IL/DPhzbqdY+IUM1qNAyS7tUiEdZbICUBEv9Qbo9oDAoGAORnY
|
||||
cSGZaumoGsAPNMRvqmQ2JYSO4c3RwGvI4RBeJV+LlthFn2Mj7h5tnCA8dkwkytBQ
|
||||
yeo6vpnrnf7B018bppxLuKBAekt8oP2biUvYxoGopp6N4kEhDVRdIxwcIHhbMoRp
|
||||
VG8VbBmiBUBbjcMIdVdeqLSme9UvclMdz/C7uzUCgYEAzD2dvoJSu9zfo3fACw1X
|
||||
te53Rvr6Kq6DTw0SS4UrAnxlhX9TwD9gymIrR9rxnC+UPwfMwWMdpYqY+Da4sF1G
|
||||
uwd+nyZ4iFNq+lbLii9rmJ58h03+uo4lUXWTrAik6qOF1ztIfYKWFrOyBQmxjWwo
|
||||
HnUinudalENFJYZRwSUAtXo=
|
||||
-----END PRIVATE KEY-----
|
||||
Reference in new issue
Block a user