2026-08-07_13-20-43_CEST

2026-08-07 13:20:43 +02:00
parent a06b950661
commit 75440a83b0
5 changed files with 922 additions and 4 deletions
@@ -0,0 +1,396 @@
 # CDN Update Script
 This document describes a reference implementation of the `cdn-update.sh` automation script for the Community CDN architecture.
 The script is designed to:
 * Download the signed control file
 * Verify its signature
 * Validate JSON syntax
 * Generate fail2ban configuration
 * Generate nginx configuration
 * Select an available origin
 * Synchronize content using rsync
 * Export Prometheus metrics
 * Fail safely when configuration validation fails
 ---
 # Reference Script
 ```bash
 #!/usr/bin/env bash
 set -euo pipefail
 ###############################################################################
 # Configuration
 ###############################################################################
 BASE="/var/lib/cdn"
 CONFIG_DIR="${BASE}/config"
 CONTENT_DIR="${BASE}/content"
 METRICS_DIR="${BASE}/metrics"
 CONTROL_URL="https://control.example.org/hpr.ccdn.settings.json"
 SIG_URL="https://control.example.org/hpr.ccdn.settings.json.minisig"
 PUBKEY="RWQxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 CONTROL_FILE="${CONFIG_DIR}/hpr.ccdn.settings.json"
 SIG_FILE="${CONFIG_DIR}/hpr.ccdn.settings.json.minisig"
 NGINX_GEN="/etc/nginx/conf.d/cdn-generated.conf"
 FAIL2BAN_JAIL="/etc/fail2ban/jail.d/cdn-generated.local"
 METRICS_FILE="${METRICS_DIR}/cdn.prom"
 TMPDIR="$(mktemp -d)"
 ###############################################################################
 # Metrics helper
 ###############################################################################
 metric_write() {
    cat > "${METRICS_FILE}" <<EOF
 cdn_last_sync_timestamp ${LAST_SYNC_TIMESTAMP:-0}
 cdn_sync_success ${SYNC_SUCCESS:-0}
 cdn_sync_duration_seconds ${SYNC_DURATION:-0}
 cdn_invalid_requests_total ${INVALID_REQUESTS:-0}
 cdn_active_origin{origin="${ACTIVE_ORIGIN:-none}"} 1
 EOF
 }
 ###############################################################################
 # Cleanup
 ###############################################################################
 cleanup() {
    rm -rf "${TMPDIR}"
 }
 trap cleanup EXIT
 ###############################################################################
 # Download control file
 ###############################################################################
 echo "Downloading control file..."
 curl -fsSL \
    -o "${TMPDIR}/hpr.ccdn.settings.json" \
    "${CONTROL_URL}"
 curl -fsSL \
    -o "${TMPDIR}/hpr.ccdn.settings.json.minisig" \
    "${SIG_URL}"
 ###############################################################################
 # Verify signature
 ###############################################################################
 echo "Verifying signature..."
 minisign \
    -Vm "${TMPDIR}/hpr.ccdn.settings.json" \
    -P "${PUBKEY}" \
    -x "${TMPDIR}/hpr.ccdn.settings.json.minisig"
 ###############################################################################
 # Validate JSON
 ###############################################################################
 jq empty "${TMPDIR}/hpr.ccdn.settings.json"
 install -m 0644 \
    "${TMPDIR}/hpr.ccdn.settings.json" \
    "${CONTROL_FILE}"
 install -m 0644 \
    "${TMPDIR}/hpr.ccdn.settings.json.minisig" \
    "${SIG_FILE}"
 ###############################################################################
 # Load values
 ###############################################################################
 mapfile -t ORIGINS < <(
    jq -r '.origins[]' "${CONTROL_FILE}"
 )
 RSYNC_INTERVAL=$(
    jq -r '.rsync_interval_hours' "${CONTROL_FILE}"
 )
 MAXRETRY=$(
    jq -r '.fail2ban.maxretry' "${CONTROL_FILE}"
 )
 FINDTIME=$(
    jq -r '.fail2ban.findtime' "${CONTROL_FILE}"
 )
 BANTIME=$(
    jq -r '.fail2ban.bantime' "${CONTROL_FILE}"
 )
 ###############################################################################
 # Generate fail2ban configuration
 ###############################################################################
 echo "Generating fail2ban configuration..."
 ADMIN_IPS=$(
    jq -r '.admin_ips[]?' "${CONTROL_FILE}" \
        | tr '\n' ' '
 )
 cat > "${FAIL2BAN_JAIL}" <<EOF
 [nginx-invalid]
 enabled = true
 maxretry = ${MAXRETRY}
 findtime = ${FINDTIME}
 bantime = ${BANTIME}
 ignoreip = 127.0.0.1 ${ADMIN_IPS}
 EOF
 systemctl reload fail2ban
 ###############################################################################
 # Select active origin
 ###############################################################################
 ACTIVE_ORIGIN=""
 for ORIGIN in "${ORIGINS[@]}"
 do
    if ssh \
        -o BatchMode=yes \
        -o ConnectTimeout=5 \
        "${ORIGIN}" \
        true
    then
        ACTIVE_ORIGIN="${ORIGIN}"
        break
    fi
 done
 if [ -z "${ACTIVE_ORIGIN}" ]
 then
    echo "No origin available"
    exit 1
 fi
 ###############################################################################
 # Generate nginx configuration
 ###############################################################################
 echo "Generating nginx config..."
 EXT_REGEX=$(
    jq -r '.allowed_extensions[]' "${CONTROL_FILE}" \
    | paste -sd'|' -
 )
 cat > "${NGINX_GEN}" <<EOF
 autoindex off;
 location ~ ^/eps/hpr[0-9]{4}/hpr[0-9]{4}\.(${EXT_REGEX})\$ {
    root ${CONTENT_DIR}/public_html;
 }
 location = /robots.txt {
    root ${CONTENT_DIR}/public_html;
 }
 location = /favicon.ico {
    root ${CONTENT_DIR}/public_html;
 }
 location / {
    access_log /var/log/nginx/invalid_requests.log;
    return 404;
 }
 EOF
 nginx -t
 systemctl reload nginx
 ###############################################################################
 # Rsync
 ###############################################################################
 echo "Starting rsync..."
 START_TIME=$(date +%s)
 if rsync \
    -az \
    --delete-delay \
    rsyncuser@"${ACTIVE_ORIGIN}":/srv/content/ \
    "${CONTENT_DIR}/"
 then
    END_TIME=$(date +%s)
    LAST_SYNC_TIMESTAMP="${END_TIME}"
    SYNC_DURATION=$((END_TIME - START_TIME))
    SYNC_SUCCESS=1
 else
    LAST_SYNC_TIMESTAMP=$(date +%s)
    SYNC_DURATION=0
    SYNC_SUCCESS=0
 fi
 ###############################################################################
 # Invalid request metric
 ###############################################################################
 INVALID_REQUESTS=$(
    wc -l \
    < /var/log/nginx/invalid_requests.log \
    || echo 0
 )
 ###############################################################################
 # Write metrics
 ###############################################################################
 metric_write
 echo "Update complete"
 ```
 ---
 # Recommended Enhancements
 ## Atomic Configuration Updates
 Generate temporary configuration files first and only replace active files after validation succeeds.
 Example workflow:
 1. Generate configuration in a temporary directory.
 2. Run `nginx -t`.
 3. Replace production configuration.
 4. Reload nginx.
 This prevents broken configuration from affecting service availability.
 ---
 ## SSH Host Key Pinning
 Use a dedicated `known_hosts` file.
 Example:
 ```bash
 ssh \
  -o UserKnownHostsFile=/etc/cdn/known_hosts \
  -o StrictHostKeyChecking=yes
 ```
 This protects against origin impersonation.
 ---
 ## Restrict Synchronized Content
 Limit rsync to approved file types.
 Example:
 ```bash
 rsync \
  -az \
  --delete-delay \
  --include='*/' \
  --include='*.mp3' \
  --include='*.ogg' \
  --include='*.opus' \
  --include='*.txt' \
  --include='*.json' \
  --exclude='*' \
  rsyncuser@origin:/srv/content/ \
  /var/lib/cdn/content/
 ```
 This prevents accidental synchronization of unexpected files.
 ---
 ## Cached Control Files
 If the control server is temporarily unavailable:
 * Continue serving content
 * Continue using the last verified control file
 * Retry on the next scheduled execution
 Nodes should never accept an unsigned replacement file.
 ---
 ## Version-Based Synchronization
 Store the last control file version.
 Only perform a full synchronization when:
 * The control file version changes
 * `force_full_rsync` is enabled
 This reduces unnecessary origin traffic.
 ---
 ## Fail2ban Dynamic Ban Includes
 Generate a separate include file for:
 * Immediate IP bans
 * Dynamic blocklists
 Avoid rewriting the primary jail configuration on every update.
 ---
 # Example Cron Schedule
 ```cron
 */5 * * * * /usr/local/bin/cdn-update.sh
 ```
 This provides:
 * Control file refresh every 5 minutes
 * Automatic failover detection
 * Automatic configuration updates
 * Regular synchronization scheduling
 ---
 # Operational Flow
 1. Download control file.
 2. Verify minisign signature.
 3. Validate JSON.
 4. Generate fail2ban configuration.
 5. Generate nginx configuration.
 6. Select active origin.
 7. Synchronize content.
 8. Export Prometheus metrics.
 9. Exit successfully.
 If any validation step fails, the script exits without modifying the running configuration.
 This fail-closed behavior helps ensure that only authenticated, valid configuration changes are applied to CDN nodes.
@@ -152,11 +152,50 @@ Example:
 The control file must be signed.
-Recommended:
+# minisign
 ## Admin
 ### Key Generation
 ```
 minisign_date="$( \date --universal +%s )"
 echo "Generating keys with date \"${minisign_date}\""
 # -G     Generate a new key pair
 # -P <pubkey>
         Public key, as a base64 string
 # -s <seckey_file>
         Secret key file (default: ~/.minisign/minisign.key
 minisign -G -p hpr.minisign.${minisign_date}.pubkey -s hpr.minisign.${minisign_date}.seckey
 ```
 ### Document Signing
 ```
 # -S     Sign files
 # -m <file>
 #        File to sign/verify
 # -t <comment>
 #        Add a one-line trusted comment
 # -x <sig_file>
 #        Signature file (default: <file>.minisig)
 # -P <pubkey>
         Public key, as a base64 string
 minisign_date=1785664421
 minisign -Sm hpr.ccdn.settings.json -x hpr.ccdn.settings.json.minisig -p hpr.minisign.${minisign_date}.pubkey -t 'Last updated on $( \date --universal +%Y-%m-%dT%H:%M:%SZ_%A ) ($( \date --universal +%s )) by $USER'
 ```
 ## Client
 ```
 minisign_date="$( \date --universal +%s )"
 echo "Generating keys with date \"${minisign_date}\""
 minisign -G -p hpr.minisign.${minisign_date}.pubkey -s hpr.minisign.${minisign_date}.seckey
 ```
 * minisign
 * signify
 * GnuPG
 Nodes must reject unsigned or invalid control files.
@@ -0,0 +1,268 @@
 # ccdn_architecture_upgrades.md
 # CCDN Architecture Upgrade Ideas
 This document lists potential improvements to the CCDN architecture that were identified during a design review. None of these are considered essential for the initial release; they are intended as future enhancements once the core system is stable.
 The philosophy remains unchanged:
 * Keep the system simple.
 * Prefer proven Unix tools.
 * Avoid unnecessary dependencies.
 * Preserve the stateless nature of edge nodes.
 ---
 ## 1. Atomic Content Deployment
 **Priority:** High
 Instead of synchronising directly into the live content directory, use a staging directory.
 Suggested workflow:
 1. Download the signed control file.
 2. Verify its signature.
 3. Synchronise content into a staging directory.
 4. Perform validation.
 5. Atomically rename the staging directory into production.
 6. Reload nginx if required.
 Benefits:
 * Prevents partially updated content from being served.
 * Makes interrupted synchronisations harmless.
 * Provides clean rollback behaviour if validation fails.
 ---
 ## 2. Content Manifest
 **Priority:** High
 Publish a manifest alongside the content containing:
 * filename
 * size
 * SHA-256 checksum
 * modification time
 After synchronisation, each node can verify downloaded files against the manifest.
 Benefits:
 * Detects storage corruption.
 * Detects incomplete synchronisations.
 * Simplifies troubleshooting.
 * Provides confidence that mirrors contain identical content.
 ---
 ## 3. Configuration Versioning
 **Priority:** Medium
 Include a version number or timestamp in the signed control file.
 Example:
 ```json
 {
    "version": 14
 }
 ```
 or
 ```json
 {
    "generated": "2026-08-01T12:00:00Z"
 }
 ```
 Benefits:
 * Easier troubleshooting.
 * Prevents accidental rollback.
 * Makes monitoring simpler.
 ---
 ## 4. Configuration Expiry
 **Priority:** Medium
 Include an expiry timestamp in the signed control file.
 Example:
 ```json
 {
    "valid_until": "2026-09-01T00:00:00Z"
 }
 ```
 Nodes should continue serving existing content if the configuration expires, but generate warnings or alerts so administrators know updates are no longer being received.
 Benefits:
 * Detects long-term communication failures.
 * Helps identify replay attacks using stale configurations.
 ---
 ## 5. Health Endpoint
 **Priority:** Medium
 Expose a small JSON document such as:
 ```
 /health.json
 ```
 Example information:
 * configuration version
 * last successful synchronisation
 * current origin
 * disk usage
 * software version
 * node identifier
 Benefits:
 * Simplifies monitoring.
 * Easy integration with Prometheus or external monitoring.
 * Useful during troubleshooting.
 ---
 ## 6. Disk Space Protection
 **Priority:** Medium
 Before synchronisation, verify sufficient free disk space exists.
 If available space falls below a configured threshold, abort the update and generate an alert.
 Benefits:
 * Prevents failed deployments.
 * Protects nodes from filling the filesystem.
 ---
 ## 7. SSH Restrictions
 **Priority:** Medium
 Restrict the rsync account so it cannot obtain an interactive shell.
 Possible approaches include:
 * `rrsync`
 * `ForceCommand`
 * `command=` restrictions in `authorized_keys`
 Benefits:
 * Reduces the impact of a compromised SSH key.
 * Limits access strictly to file synchronisation.
 ---
 ## 8. Signing Key Rotation
 **Priority:** Low
 Support publishing both the current and next public signing keys.
 This allows new keys to be distributed before they become active.
 Benefits:
 * Simplifies planned key rotation.
 * Avoids emergency replacement procedures.
 ---
 ## 9. Split Configuration Files
 **Priority:** Low
 If the control file becomes large, consider splitting it into independently signed files.
 For example:
 * origins.json
 * sync.json
 * nginx.json
 * security.json
 Benefits:
 * Easier maintenance.
 * Smaller updates.
 * Simpler reviews.
 This is not recommended until the configuration grows significantly.
 ---
 ## 10. Declarative Configuration Only
 **Priority:** Ongoing
 The signed configuration should describe desired system state rather than commands to execute.
 For example:
 Prefer:
 * sync interval
 * fail2ban settings
 * nginx options
 * origin list
 Avoid:
 * arbitrary shell commands
 * remote script execution
 Benefits:
 * Smaller attack surface.
 * Easier auditing.
 * More predictable behaviour.
 ---
 # Future Enhancements
 These ideas are intentionally outside the scope of the first implementation but may become useful if CCDN grows.
 * Delta manifests for large libraries.
 * Geographic origin selection.
 * Optional peer-to-peer mirror synchronisation.
 * Compression for text-based metadata.
 * Signed content release tags.
 * Read-only content mounts between updates.
 ---
 # Deliberately Out of Scope
 The following technologies are intentionally excluded from the CCDN design:
 * Kubernetes
 * Docker Swarm
 * Redis
 * PostgreSQL
 * Elasticsearch
 * Message queues
 * Dynamic service discovery
 * Distributed databases
 * Complex orchestration systems
 The goal is to keep CCDN easy to understand, easy to operate, and easy to recover by relying on mature Unix tooling rather than additional infrastructure.
@@ -0,0 +1,213 @@
 # Community Content Delivery Network Docker nodes
 ## Background
 We wish to deploy a Community CDN that is easy to deploy and maintain.
 This can be achieved by providing a Community Managed docker nodes that is regularly updated.
 Changes can be applied to the GitTea, and the mirror nodes should pick up the changes.
 ## Removal of node
 - Remove from central DNS
 - Remove from monitoring
 - Remove from allow lists
 - Remove from authorized_keys
 ## 
 - Addition of new nodes
 - Removal of old nodes
 Path Validation Strategy
 This is where your design differs from a normal web server.
 You know:
 episode numbers
 valid extensions
 exact file list
 Therefore every request can be validated.
 Option 1 (recommended)
 Allow nginx to serve files normally.
 If file does not exist:
 error_page 404 = @invalid_request;
 location @invalid_request {
    access_log /var/log/nginx/invalid-paths.log;
    return 404;
 }
 This scales well even with tens of thousands of files.
 Prevent Directory Browsing
 autoindex off;
 location / {
    try_files $uri =404;
 }
 No listings.
 No traversal.
 No guessing directories.
 thank you - I have refined the requirements now so ignore what I said earlier.
 We are building a private community content delivery network, with nodes that serves files that typically change once a day.
 ## Hardware Requirements
 Requirements for Hosting
 - 24/7 Home Service
 - fixed IP address
 - unlimited bandwidth
 - fast > 500mb/sec upload
 - large > 5T of storage
 - permission from your ISP to run a web server
 - Contact information know to the Janitors
 - Optional: UPS
 ## Software updates
 The nodes should ideally run on both [Docker](https://en.wikipedia.org/wiki/Docker_%28software%29) and [Podman](https://en.wikipedia.org/wiki/Podman)
 It should run Debian `debian:stable-slim` 
 All software MUST be running verifiable Free Libre Open Source software with links to the license.
 The nodes should have as little utilities as possible to do it's job and in order to reduce the attack surface.
 The amount of traffic is about 5Mb/sec with a load of 2 TPS
 There will be many nodes, that will come and go over time.
 The nodes will be added to DNS so the load can be shared.
 The origin server requirements will be dictated by the needs of the nodes.
 Nodes will monitor a rss control channel from the origin server for instructions.
 The RSS file will contain a link to a json file.
 The json file will contain 
 - the fail2ban settings, list of ip addresses, maxretry, findtime and bantime
 - force full rsync Boolean flag - false
 - hours between rsync eg every 3 hours
 - list of origin servers
 - list of Admin IP addresses
 - list of IP addresses to ban immediately 
 - list of useragents to ban immediately 
 If the origin server is not available then the nodes should connect to the next origin server in the list.
 nodes will synchronize the files from a origin server using rsync over ssh.
 A full rsync is sufficient to ensure files are copied correctly.
 A full rsync will be done several times a day, eg every 3 hours - from settings
 atomic updates are not required as only new files are added.
 There are thousands of files
 The files will come from a well known directory eg `public_html/eps/hpr${ep_num}/hpr${ep_num}*.${extension}`
 The `ep_num` will be a digit from 0001 to 9999
 The `extension` will be from a well know list from the origin updated perhaps once a year
 nodes will then serve the files using `nginx`
 `nginx` must restrict access to files outside the well known files
 A list of well known files that clients require (favorite icon, robots.txt, etc) will be available from the origin server.
 The files we wish to serve are all Creative Commons Licensed so can be shared.
 Random browsing is not allowed.
 All file paths are known, and are provided by the origin server.
 There should never be any request for a file outside the well known paths.
 Any attempt to access any file outside the known paths should be logged.
 Based on a configuration provided by the origin server, the server will run failtoban
 repeat offenders will be banned for 7 days.
 The Admin IP addresses should never be banned
 traffic should be secured with let's encrypt of equivalent tls transport
 clients can request the files without tls
 The nodes public ip address will be placed on a allow list for ssh
 Monitoring should be available using Prometheus or similar.
                    +----------------+
                    | Origin Server A|
                    +--------+-------+
                             |
                    RSS Control Feed
                             |
                    JSON Configuration
                             |
          +------------------+------------------+
          |                                     |
          v                                     v
 +----------------+                   +----------------+
 | Edge Node 1    |                   | Edge Node 2    |
 |                |                   |                |
 | nginx          |                   | nginx          |
 | rsync          |                   | rsync          |
 | fail2ban       |                   | fail2ban       |
 | node-agent     |                   | node-agent     |
 | prometheus exp.|                   | prometheus exp.|
 +--------+-------+                   +--------+-------+
         |                                     |
         +----------------+--------------------+
                          |
                        DNS RR
                          |
                       Clients
@@ -0,0 +1,2 @@
 untrusted comment: minisign public key 85F68B90C4D786B9
 RWS5htfEkIv2hSLt8UJmbUF4MV5P0JX+IcHq354RNF6wgC/bO/M3OxqU
		`@@ -0,0 +1,2 @@`
							`untrusted comment: minisign public key 85F68B90C4D786B9`
							`RWS5htfEkIv2hSLt8UJmbUF4MV5P0JX+IcHq354RNF6wgC/bO/M3OxqU`