# shellcheck shell=bash # cert-publish-lib.sh - safe publication of HAProxy certificate bundles. # # This file is SOURCED, never executed (hence no shebang / no exec bit). # # Why this exists # --------------- # renew-certificates.sh and sync-certificates.sh used to publish a bundle with # # cat "$CERT_FILE" "$KEY_FILE" > "$COMBINED_FILE" # # where $COMBINED_FILE is the LIVE pem HAProxy is serving right now. The shell # truncates the destination to zero bytes when it sets up the redirect, BEFORE # cat ever runs, so any failure after that point (unreadable source, ENOSPC, # container killed mid-write) leaves a zero-length or key-less pem in place. # Checking cat's exit status does not help: the damage is already done. # # That matters more here than for an ordinary file because HAProxy loads # $SSL_CERTS_DIR as a DIRECTORY: # # bind 0.0.0.0:443 ssl crt /etc/haproxy/certs # # It tries to load *every* file in that directory, and one unloadable file # fails the whole bind - i.e. HTTPS goes down for every customer on the host. # # Two consequences drive the design below: # 1. Assemble somewhere else and rename into place, so the live pem is either # the old bundle or the new one and never a half-written one. # 2. NEVER create a temp file, .tmp, .backup or any other non-final file # inside $SSL_CERTS_DIR. Staging and backups live in SIBLING directories. # # Directory layout (kept identical to the Python half in haproxy_manager.py): # staging: $(dirname $SSL_CERTS_DIR)/cert-staging [$CERT_STAGING_DIR] # backups: $(dirname $SSL_CERTS_DIR)/cert-backups [$CERT_BACKUP_DIR] # Both siblings of the certs dir, so they are on the same filesystem and the # final mv is a rename(2) - atomic. If the mv ever fails (EXDEV because someone # mounted the certs dir separately, permissions, ...) we FAIL LOUDLY and leave # the live pem alone. There is deliberately no "just write it directly" path. # Logging: the callers define their own log_info/log_error. Only provide # fallbacks so this library is usable standalone (e.g. from a test or a shell). declare -F log_info >/dev/null || log_info() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $*" } declare -F log_error >/dev/null || log_error() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*" >&2 } # log_warn is not part of the callers' vocabulary; route it through log_info # with a loud prefix so it lands in the main log without tripping the # error-log monitors (scripts/monitor-errors.sh) for non-fatal conditions. declare -F log_warn >/dev/null || log_warn() { log_info "WARNING: $*" } cert_staging_dir() { if [ -n "${CERT_STAGING_DIR:-}" ]; then echo "$CERT_STAGING_DIR" else echo "$(dirname "${SSL_CERTS_DIR:-/etc/haproxy/certs}")/cert-staging" fi } cert_backup_dir() { if [ -n "${CERT_BACKUP_DIR:-}" ]; then echo "$CERT_BACKUP_DIR" else echo "$(dirname "${SSL_CERTS_DIR:-/etc/haproxy/certs}")/cert-backups" fi } # cert_bundle_valid FILE # # Returns 0 if FILE is publishable as an HAProxy pem bundle. # # Layer 1 (MANDATORY, pure shell/grep, always available): structural checks. # A missing or failed structural check is a HARD FAIL. This layer is # what actually covers the truncation / partial-write / key-less # failure modes this library exists to prevent. # Layer 2 (BEST EFFORT): cryptographic pairing via the openssl CLI. # If openssl runs and says the cert and key do not match, that is a # HARD FAIL. If the openssl BINARY IS ABSENT we log a loud warning and # accept the bundle on the structural checks alone. # # Rationale for not hard-failing on a missing checker: the container # image (see Dockerfile) installs haproxy, certbot and socat, but not # necessarily the openssl CLI. Refusing to publish when the checker is # missing would stall every renewal fleet-wide and let certificates # expire - a guaranteed outage - which is strictly worse than the risk # it prevents, since a mismatched pair can only arise from a # mis-assembled source tree, whereas the truncation modes we are # actually defending against are fully covered by layer 1. cert_bundle_valid() { local file="$1" if [ -z "$file" ]; then log_error "cert_bundle_valid: no file given" return 1 fi if [ ! -f "$file" ]; then log_error "Certificate bundle $file does not exist (or is not a regular file)" return 1 fi if [ ! -s "$file" ]; then log_error "Certificate bundle $file is empty" return 1 fi # --- layer 1: structure ------------------------------------------------- if ! grep -qF -- '-----BEGIN CERTIFICATE-----' "$file"; then log_error "Certificate bundle $file contains no certificate block" return 1 fi if ! grep -qF -- '-----END CERTIFICATE-----' "$file"; then log_error "Certificate bundle $file has an unterminated certificate block (truncated?)" return 1 fi local key_begin key_end key_begin="$(grep -m1 -oE -- '-----BEGIN (RSA |EC )?PRIVATE KEY-----' "$file")" if [ -z "$key_begin" ]; then log_error "Certificate bundle $file contains no private key block" return 1 fi key_end="${key_begin/BEGIN/END}" if ! grep -qF -- "$key_end" "$file"; then log_error "Certificate bundle $file has an unterminated private key block (truncated?)" return 1 fi # --- layer 2: cert/key pairing ------------------------------------------ if ! command -v openssl >/dev/null 2>&1; then # Once per process: this fires per domain otherwise, and a renewal run # walks every certificate on the host. if [ -z "${_CERT_OPENSSL_WARNED:-}" ]; then _CERT_OPENSSL_WARNED=1 log_warn "openssl binary not found - SKIPPING the cert/key pairing check" \ "(openssl x509 -pubkey vs openssl pkey -pubout);" \ "certificate bundles are being published on structural checks alone" fi return 0 fi local cert_pub key_pub # /dev/null /dev/null /dev/null base="$(basename "$dest_file")" tmp="$(mktemp "${staging_dir}/${base}.XXXXXX" 2>/dev/null)" if [ -z "$tmp" ] || [ ! -f "$tmp" ]; then log_error "cert_publish: cannot create a staging file in $staging_dir" return 1 fi # 0600 while the temp file holds a private key; the final mode is matched # to the file being replaced just before the swap (see below). chmod 600 "$tmp" 2>/dev/null if ! cat "$cert_file" "$key_file" > "$tmp"; then log_error "cert_publish: failed to assemble $cert_file + $key_file (live $dest_file left untouched)" rm -f "$tmp" return 1 fi if [ ! -s "$tmp" ]; then log_error "cert_publish: assembled bundle for $dest_file is empty (live file left untouched)" rm -f "$tmp" return 1 fi # (c) never promote something HAProxy would choke on. if ! cert_bundle_valid "$tmp"; then log_error "cert_publish: assembled bundle for $dest_file failed validation (live file left untouched)" rm -f "$tmp" return 1 fi # (d) back up the bundle we are about to replace - but only if it is itself # valid. Overwriting a good backup with garbage would turn "restore the # backup" into "restore a different broken file". Same semantics as # create_backup(require_valid=True) in haproxy_manager.py. if [ -e "$dest_file" ]; then backup_dir="$(cert_backup_dir)" if cert_bundle_valid "$dest_file"; then if mkdir -p "$backup_dir"; then if ! cp -p "$dest_file" "${backup_dir}/${base}"; then log_warn "could not back up $dest_file to ${backup_dir}/${base}; publishing anyway" fi else log_warn "could not create backup directory $backup_dir; publishing without a backup" fi else log_warn "existing $dest_file is not a valid bundle - KEEPING the previous backup" \ "in $backup_dir rather than overwriting it with an unusable one" fi fi # Preserve the mode of the bundle being replaced (0644 by default, which is # what `cat > file` produced under the standard umask). mktemp gives 0600, # and mv carries the temp file's mode onto the destination, so without this # every publish would silently tighten the live pem's permissions. Changing # who can read these files is not something a write-safety fix should do as # a side effect - and it must match write_config_atomically() on the Python # side, which preserves the mode the same way. local mode mode="$(stat -c '%a' "$dest_file" 2>/dev/null)" [ -n "$mode" ] || mode=644 chmod "$mode" "$tmp" 2>/dev/null # (e) atomic swap. Same filesystem by construction; if it still fails, # stop - do not fall back to writing into the certs dir. if ! mv -f "$tmp" "$dest_file"; then log_error "cert_publish: failed to move $tmp into place as $dest_file" \ "(live file left untouched; NOT falling back to a direct write)" rm -f "$tmp" return 1 fi return 0 } # haproxy_config_ok # # Gate a reload on `haproxy -c`. Returns 0 if the config validates, or if we # cannot check (no haproxy binary) - a missing checker must not block a reload # that is otherwise needed, but a checker that says "no" always wins. haproxy_config_ok() { local cfg="${HAPROXY_CONFIG:-/etc/haproxy/haproxy.cfg}" local out rc if ! command -v haproxy >/dev/null 2>&1; then log_warn "haproxy binary not found - skipping 'haproxy -c' validation before reload" return 0 fi out="$(haproxy -c -f "$cfg" 2>&1