haproxy-manager-base/scripts/renew-certificates.sh

#!/usr/bin/env bash

# Certificate Renewal Script for HAProxy Manager
# This script runs certbot renew and copies certificates to HAProxy format

# Configuration
LOG_FILE="${LOG_FILE:-/var/log/haproxy-manager.log}"
ERROR_LOG_FILE="${ERROR_LOG_FILE:-/var/log/haproxy-manager-errors.log}"
DB_FILE="${DB_FILE:-/etc/haproxy/haproxy_config.db}"
SSL_CERTS_DIR="${SSL_CERTS_DIR:-/etc/haproxy/certs}"
LETSENCRYPT_LIVE_DIR="${LETSENCRYPT_LIVE_DIR:-/etc/letsencrypt/live}"

# Logging functions
log_info() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $*" | tee -a "$LOG_FILE"
}

log_error() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*" | tee -a "$LOG_FILE" >> "$ERROR_LOG_FILE"
}

# Safe certificate publication helpers (cert_publish / cert_bundle_valid /
# haproxy_config_ok). Sourced AFTER the log_* functions above so the library
# uses this script's logging rather than its own fallbacks.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=cert-publish-lib.sh
if [ -r "${SCRIPT_DIR}/cert-publish-lib.sh" ]; then
    . "${SCRIPT_DIR}/cert-publish-lib.sh"
else
    log_error "Missing ${SCRIPT_DIR}/cert-publish-lib.sh - refusing to touch live certificates"
    exit 1
fi

log_info "Starting certificate renewal process"

# Run certbot renewal — don't exit on failure, some certs may have
# renewed successfully even if others failed (e.g., domain no longer
# pointed here). Continue to copy/combine whatever succeeded.
CERTBOT_OUTPUT=$(certbot renew --no-random-sleep-on-renew 2>&1)
CERTBOT_EXIT=$?

if [ $CERTBOT_EXIT -eq 0 ]; then
    log_info "Certbot renewal completed successfully"
else
    log_error "Certbot renewal had failures (exit code $CERTBOT_EXIT):"
    # Log the specific failures
    echo "$CERTBOT_OUTPUT" | grep -E "Failed to renew|failure" | while read -r line; do
        log_error "  $line"
    done
    log_info "Continuing to process successfully renewed certificates..."
fi

# Copy all certificates to HAProxy format
# Ensure SSL certs directory exists
mkdir -p "$SSL_CERTS_DIR"

# Get all SSL-enabled domains from database
DOMAINS=$(find "$LETSENCRYPT_LIVE_DIR/" -mindepth 1 -maxdepth 1 -type d -printf '%f\n')

if [ -z "$DOMAINS" ]; then
    log_info "No SSL-enabled domains found"
    exit 0
fi

# Copy certificates for each domain
UPDATED=0
FAILED=0
SKIPPED_EXPIRED=0

while read -r domain; do
    CERT_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/fullchain.pem"
    KEY_FILE="${LETSENCRYPT_LIVE_DIR}/${domain}/privkey.pem"
    COMBINED_FILE="${SSL_CERTS_DIR}/${domain}.pem"

    if [ -f "$CERT_FILE" ] && cert_is_expired "$CERT_FILE"; then
        # Never publish an expired lineage. It cannot help any client and,
        # because HAProxy serves the first file claiming an SNI name, it can
        # shadow a valid bundle for a name it shares (brainjar.net, 2026-10-07).
        # Counted separately from FAILED: a dead lineage that keeps expiring is
        # reported on every run (ERROR log) without making every run "fail".
        log_error "Skipping EXPIRED certificate lineage $domain (notAfter $(cert_not_after "$CERT_FILE")) - not publishing it"
        SKIPPED_EXPIRED=$((SKIPPED_EXPIRED + 1))
    elif [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then
        # Assemble in a staging dir and rename into place. NEVER redirect into
        # $COMBINED_FILE: the shell truncates the live pem before cat runs, and
        # HAProxy loads $SSL_CERTS_DIR as a directory, so one bad file there
        # takes down the whole ssl bind. See scripts/cert-publish-lib.sh.
        if cert_publish "$CERT_FILE" "$KEY_FILE" "$COMBINED_FILE"; then
            log_info "Updated certificate for $domain"
            UPDATED=$((UPDATED + 1))
        else
            log_error "Failed to combine certificate for $domain"
            FAILED=$((FAILED + 1))
        fi
    else
        log_error "Certificate files not found for $domain"
        FAILED=$((FAILED + 1))
    fi
done <<< "$DOMAINS"

# Already-published pems that have expired since are moved out too (they
# can only shadow; see cert_retire_expired_pems).
cert_retire_expired_pems
RETIRED_EXPIRED=$CERT_RETIRED_EXPIRED

log_info "Certificate update completed: $UPDATED updated, $FAILED failed, $SKIPPED_EXPIRED expired lineage(s) skipped, $RETIRED_EXPIRED expired pem(s) retired"

# Reload HAProxy if any certificates were updated
if [ $UPDATED -gt 0 ] || [ "$RETIRED_EXPIRED" -gt 0 ]; then
    # Never reload onto unvalidated material: a reload that fails to load the
    # certs directory drops HTTPS for every site on this host.
    if ! haproxy_config_ok; then
        log_error "HAProxy configuration does not validate - refusing to reload after certificate renewal"
        exit 1
    fi

    if echo "reload" | socat stdio /tmp/haproxy-cli 2>/dev/null; then
        log_info "HAProxy reloaded successfully"
    else
        log_error "Failed to reload HAProxy"
        exit 1
    fi
fi

# A per-domain publication failure is a real failure and must be reported as
# one. This script used to `exit 0` no matter how many domains failed, so
# "0 updated, 12 failed" - a host that has completely stopped publishing
# renewals - looked identical to a clean run to cron, to
# host-renew-certificates.sh (which branches on this exit code) and to any
# external monitoring. The first visible symptom would have been certificates
# expiring. The loop above deliberately continues past a failed domain so the
# others still get published; the status is reported here instead.
if [ "$FAILED" -gt 0 ]; then
    log_error "Certificate renewal process completed with failures: $UPDATED updated, $FAILED failed"
    exit 1
fi

log_info "Certificate renewal process completed"
exit 0