2026-08-07_13-20-43_CEST

2026-08-07 13:20:43 +02:00
parent a06b950661
commit 75440a83b0
5 changed files with 922 additions and 4 deletions
@@ -0,0 +1,396 @@
+# CDN Update Script
+
+This document describes a reference implementation of the `cdn-update.sh` automation script for the Community CDN architecture.
+
+The script is designed to:
+
+* Download the signed control file
+* Verify its signature
+* Validate JSON syntax
+* Generate fail2ban configuration
+* Generate nginx configuration
+* Select an available origin
+* Synchronize content using rsync
+* Export Prometheus metrics
+* Fail safely when configuration validation fails
+
+---
+
+# Reference Script
+
+```bash
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+###############################################################################
+# Configuration
+###############################################################################
+
+BASE="/var/lib/cdn"
+
+CONFIG_DIR="${BASE}/config"
+CONTENT_DIR="${BASE}/content"
+METRICS_DIR="${BASE}/metrics"
+
+CONTROL_URL="https://control.example.org/hpr.ccdn.settings.json"
+SIG_URL="https://control.example.org/hpr.ccdn.settings.json.minisig"
+
+PUBKEY="RWQxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+CONTROL_FILE="${CONFIG_DIR}/hpr.ccdn.settings.json"
+SIG_FILE="${CONFIG_DIR}/hpr.ccdn.settings.json.minisig"
+
+NGINX_GEN="/etc/nginx/conf.d/cdn-generated.conf"
+
+FAIL2BAN_JAIL="/etc/fail2ban/jail.d/cdn-generated.local"
+
+METRICS_FILE="${METRICS_DIR}/cdn.prom"
+
+TMPDIR="$(mktemp -d)"
+
+###############################################################################
+# Metrics helper
+###############################################################################
+
+metric_write() {
+    cat > "${METRICS_FILE}" <<EOF
+cdn_last_sync_timestamp ${LAST_SYNC_TIMESTAMP:-0}
+cdn_sync_success ${SYNC_SUCCESS:-0}
+cdn_sync_duration_seconds ${SYNC_DURATION:-0}
+cdn_invalid_requests_total ${INVALID_REQUESTS:-0}
+cdn_active_origin{origin="${ACTIVE_ORIGIN:-none}"} 1
+EOF
+}
+
+###############################################################################
+# Cleanup
+###############################################################################
+
+cleanup() {
+    rm -rf "${TMPDIR}"
+}
+
+trap cleanup EXIT
+
+###############################################################################
+# Download control file
+###############################################################################
+
+echo "Downloading control file..."
+
+curl -fsSL \
+    -o "${TMPDIR}/hpr.ccdn.settings.json" \
+    "${CONTROL_URL}"
+
+curl -fsSL \
+    -o "${TMPDIR}/hpr.ccdn.settings.json.minisig" \
+    "${SIG_URL}"
+
+###############################################################################
+# Verify signature
+###############################################################################
+
+echo "Verifying signature..."
+
+minisign \
+    -Vm "${TMPDIR}/hpr.ccdn.settings.json" \
+    -P "${PUBKEY}" \
+    -x "${TMPDIR}/hpr.ccdn.settings.json.minisig"
+
+###############################################################################
+# Validate JSON
+###############################################################################
+
+jq empty "${TMPDIR}/hpr.ccdn.settings.json"
+
+install -m 0644 \
+    "${TMPDIR}/hpr.ccdn.settings.json" \
+    "${CONTROL_FILE}"
+
+install -m 0644 \
+    "${TMPDIR}/hpr.ccdn.settings.json.minisig" \
+    "${SIG_FILE}"
+
+###############################################################################
+# Load values
+###############################################################################
+
+mapfile -t ORIGINS < <(
+    jq -r '.origins[]' "${CONTROL_FILE}"
+)
+
+RSYNC_INTERVAL=$(
+    jq -r '.rsync_interval_hours' "${CONTROL_FILE}"
+)
+
+MAXRETRY=$(
+    jq -r '.fail2ban.maxretry' "${CONTROL_FILE}"
+)
+
+FINDTIME=$(
+    jq -r '.fail2ban.findtime' "${CONTROL_FILE}"
+)
+
+BANTIME=$(
+    jq -r '.fail2ban.bantime' "${CONTROL_FILE}"
+)
+
+###############################################################################
+# Generate fail2ban configuration
+###############################################################################
+
+echo "Generating fail2ban configuration..."
+
+ADMIN_IPS=$(
+    jq -r '.admin_ips[]?' "${CONTROL_FILE}" \
+        | tr '\n' ' '
+)
+
+cat > "${FAIL2BAN_JAIL}" <<EOF
+[nginx-invalid]
+enabled = true
+
+maxretry = ${MAXRETRY}
+findtime = ${FINDTIME}
+bantime = ${BANTIME}
+
+ignoreip = 127.0.0.1 ${ADMIN_IPS}
+EOF
+
+systemctl reload fail2ban
+
+###############################################################################
+# Select active origin
+###############################################################################
+
+ACTIVE_ORIGIN=""
+
+for ORIGIN in "${ORIGINS[@]}"
+do
+    if ssh \
+        -o BatchMode=yes \
+        -o ConnectTimeout=5 \
+        "${ORIGIN}" \
+        true
+    then
+        ACTIVE_ORIGIN="${ORIGIN}"
+        break
+    fi
+done
+
+if [ -z "${ACTIVE_ORIGIN}" ]
+then
+    echo "No origin available"
+    exit 1
+fi
+
+###############################################################################
+# Generate nginx configuration
+###############################################################################
+
+echo "Generating nginx config..."
+
+EXT_REGEX=$(
+    jq -r '.allowed_extensions[]' "${CONTROL_FILE}" \
+    | paste -sd'|' -
+)
+
+cat > "${NGINX_GEN}" <<EOF
+autoindex off;
+
+location ~ ^/eps/hpr[0-9]{4}/hpr[0-9]{4}\.(${EXT_REGEX})\$ {
+    root ${CONTENT_DIR}/public_html;
+}
+
+location = /robots.txt {
+    root ${CONTENT_DIR}/public_html;
+}
+
+location = /favicon.ico {
+    root ${CONTENT_DIR}/public_html;
+}
+
+location / {
+    access_log /var/log/nginx/invalid_requests.log;
+    return 404;
+}
+EOF
+
+nginx -t
+systemctl reload nginx
+
+###############################################################################
+# Rsync
+###############################################################################
+
+echo "Starting rsync..."
+
+START_TIME=$(date +%s)
+
+if rsync \
+    -az \
+    --delete-delay \
+    rsyncuser@"${ACTIVE_ORIGIN}":/srv/content/ \
+    "${CONTENT_DIR}/"
+then
+
+    END_TIME=$(date +%s)
+
+    LAST_SYNC_TIMESTAMP="${END_TIME}"
+    SYNC_DURATION=$((END_TIME - START_TIME))
+    SYNC_SUCCESS=1
+
+else
+
+    LAST_SYNC_TIMESTAMP=$(date +%s)
+    SYNC_DURATION=0
+    SYNC_SUCCESS=0
+
+fi
+
+###############################################################################
+# Invalid request metric
+###############################################################################
+
+INVALID_REQUESTS=$(
+    wc -l \
+    < /var/log/nginx/invalid_requests.log \
+    || echo 0
+)
+
+###############################################################################
+# Write metrics
+###############################################################################
+
+metric_write
+
+echo "Update complete"
+```
+
+---
+
+# Recommended Enhancements
+
+## Atomic Configuration Updates
+
+Generate temporary configuration files first and only replace active files after validation succeeds.
+
+Example workflow:
+
+1. Generate configuration in a temporary directory.
+2. Run `nginx -t`.
+3. Replace production configuration.
+4. Reload nginx.
+
+This prevents broken configuration from affecting service availability.
+
+---
+
+## SSH Host Key Pinning
+
+Use a dedicated `known_hosts` file.
+
+Example:
+
+```bash
+ssh \
+  -o UserKnownHostsFile=/etc/cdn/known_hosts \
+  -o StrictHostKeyChecking=yes
+```
+
+This protects against origin impersonation.
+
+---
+
+## Restrict Synchronized Content
+
+Limit rsync to approved file types.
+
+Example:
+
+```bash
+rsync \
+  -az \
+  --delete-delay \
+  --include='*/' \
+  --include='*.mp3' \
+  --include='*.ogg' \
+  --include='*.opus' \
+  --include='*.txt' \
+  --include='*.json' \
+  --exclude='*' \
+  rsyncuser@origin:/srv/content/ \
+  /var/lib/cdn/content/
+```
+
+This prevents accidental synchronization of unexpected files.
+
+---
+
+## Cached Control Files
+
+If the control server is temporarily unavailable:
+
+* Continue serving content
+* Continue using the last verified control file
+* Retry on the next scheduled execution
+
+Nodes should never accept an unsigned replacement file.
+
+---
+
+## Version-Based Synchronization
+
+Store the last control file version.
+
+Only perform a full synchronization when:
+
+* The control file version changes
+* `force_full_rsync` is enabled
+
+This reduces unnecessary origin traffic.
+
+---
+
+## Fail2ban Dynamic Ban Includes
+
+Generate a separate include file for:
+
+* Immediate IP bans
+* Dynamic blocklists
+
+Avoid rewriting the primary jail configuration on every update.
+
+---
+
+# Example Cron Schedule
+
+```cron
+*/5 * * * * /usr/local/bin/cdn-update.sh
+```
+
+This provides:
+
+* Control file refresh every 5 minutes
+* Automatic failover detection
+* Automatic configuration updates
+* Regular synchronization scheduling
+
+---
+
+# Operational Flow
+
+1. Download control file.
+2. Verify minisign signature.
+3. Validate JSON.
+4. Generate fail2ban configuration.
+5. Generate nginx configuration.
+6. Select active origin.
+7. Synchronize content.
+8. Export Prometheus metrics.
+9. Exit successfully.
+
+If any validation step fails, the script exits without modifying the running configuration.
+
+This fail-closed behavior helps ensure that only authenticated, valid configuration changes are applied to CDN nodes.
@@ -152,11 +152,50 @@ Example:

 The control file must be signed.

-Recommended:
+# minisign
+
+## Admin
+
+### Key Generation
+
+```
+minisign_date="$( \date --universal +%s )"
+echo "Generating keys with date \"${minisign_date}\""
+# -G     Generate a new key pair
+# -P <pubkey>
+         Public key, as a base64 string
+# -s <seckey_file>
+         Secret key file (default: ~/.minisign/minisign.key
+minisign -G -p hpr.minisign.${minisign_date}.pubkey -s hpr.minisign.${minisign_date}.seckey
+```
+### Document Signing
+
+```
+# -S     Sign files
+# -m <file>
+#        File to sign/verify
+# -t <comment>
+#        Add a one-line trusted comment
+# -x <sig_file>
+#        Signature file (default: <file>.minisig)
+# -P <pubkey>
+         Public key, as a base64 string
+
+minisign_date=1785664421
+minisign -Sm hpr.ccdn.settings.json -x hpr.ccdn.settings.json.minisig -p hpr.minisign.${minisign_date}.pubkey -t 'Last updated on $( \date --universal +%Y-%m-%dT%H:%M:%SZ_%A ) ($( \date --universal +%s )) by $USER'
+
+```
+
+
+## Client
+
+```
+minisign_date="$( \date --universal +%s )"
+echo "Generating keys with date \"${minisign_date}\""
+minisign -G -p hpr.minisign.${minisign_date}.pubkey -s hpr.minisign.${minisign_date}.seckey
+```
+

-* minisign
-* signify
-* GnuPG

 Nodes must reject unsigned or invalid control files.

@@ -0,0 +1,268 @@
+# ccdn_architecture_upgrades.md
+
+# CCDN Architecture Upgrade Ideas
+
+This document lists potential improvements to the CCDN architecture that were identified during a design review. None of these are considered essential for the initial release; they are intended as future enhancements once the core system is stable.
+
+The philosophy remains unchanged:
+
+* Keep the system simple.
+* Prefer proven Unix tools.
+* Avoid unnecessary dependencies.
+* Preserve the stateless nature of edge nodes.
+
+---
+
+## 1. Atomic Content Deployment
+
+**Priority:** High
+
+Instead of synchronising directly into the live content directory, use a staging directory.
+
+Suggested workflow:
+
+1. Download the signed control file.
+2. Verify its signature.
+3. Synchronise content into a staging directory.
+4. Perform validation.
+5. Atomically rename the staging directory into production.
+6. Reload nginx if required.
+
+Benefits:
+
+* Prevents partially updated content from being served.
+* Makes interrupted synchronisations harmless.
+* Provides clean rollback behaviour if validation fails.
+
+---
+
+## 2. Content Manifest
+
+**Priority:** High
+
+Publish a manifest alongside the content containing:
+
+* filename
+* size
+* SHA-256 checksum
+* modification time
+
+After synchronisation, each node can verify downloaded files against the manifest.
+
+Benefits:
+
+* Detects storage corruption.
+* Detects incomplete synchronisations.
+* Simplifies troubleshooting.
+* Provides confidence that mirrors contain identical content.
+
+---
+
+## 3. Configuration Versioning
+
+**Priority:** Medium
+
+Include a version number or timestamp in the signed control file.
+
+Example:
+
+```json
+{
+    "version": 14
+}
+```
+
+or
+
+```json
+{
+    "generated": "2026-08-01T12:00:00Z"
+}
+```
+
+Benefits:
+
+* Easier troubleshooting.
+* Prevents accidental rollback.
+* Makes monitoring simpler.
+
+---
+
+## 4. Configuration Expiry
+
+**Priority:** Medium
+
+Include an expiry timestamp in the signed control file.
+
+Example:
+
+```json
+{
+    "valid_until": "2026-09-01T00:00:00Z"
+}
+```
+
+Nodes should continue serving existing content if the configuration expires, but generate warnings or alerts so administrators know updates are no longer being received.
+
+Benefits:
+
+* Detects long-term communication failures.
+* Helps identify replay attacks using stale configurations.
+
+---
+
+## 5. Health Endpoint
+
+**Priority:** Medium
+
+Expose a small JSON document such as:
+
+```
+/health.json
+```
+
+Example information:
+
+* configuration version
+* last successful synchronisation
+* current origin
+* disk usage
+* software version
+* node identifier
+
+Benefits:
+
+* Simplifies monitoring.
+* Easy integration with Prometheus or external monitoring.
+* Useful during troubleshooting.
+
+---
+
+## 6. Disk Space Protection
+
+**Priority:** Medium
+
+Before synchronisation, verify sufficient free disk space exists.
+
+If available space falls below a configured threshold, abort the update and generate an alert.
+
+Benefits:
+
+* Prevents failed deployments.
+* Protects nodes from filling the filesystem.
+
+---
+
+## 7. SSH Restrictions
+
+**Priority:** Medium
+
+Restrict the rsync account so it cannot obtain an interactive shell.
+
+Possible approaches include:
+
+* `rrsync`
+* `ForceCommand`
+* `command=` restrictions in `authorized_keys`
+
+Benefits:
+
+* Reduces the impact of a compromised SSH key.
+* Limits access strictly to file synchronisation.
+
+---
+
+## 8. Signing Key Rotation
+
+**Priority:** Low
+
+Support publishing both the current and next public signing keys.
+
+This allows new keys to be distributed before they become active.
+
+Benefits:
+
+* Simplifies planned key rotation.
+* Avoids emergency replacement procedures.
+
+---
+
+## 9. Split Configuration Files
+
+**Priority:** Low
+
+If the control file becomes large, consider splitting it into independently signed files.
+
+For example:
+
+* origins.json
+* sync.json
+* nginx.json
+* security.json
+
+Benefits:
+
+* Easier maintenance.
+* Smaller updates.
+* Simpler reviews.
+
+This is not recommended until the configuration grows significantly.
+
+---
+
+## 10. Declarative Configuration Only
+
+**Priority:** Ongoing
+
+The signed configuration should describe desired system state rather than commands to execute.
+
+For example:
+
+Prefer:
+
+* sync interval
+* fail2ban settings
+* nginx options
+* origin list
+
+Avoid:
+
+* arbitrary shell commands
+* remote script execution
+
+Benefits:
+
+* Smaller attack surface.
+* Easier auditing.
+* More predictable behaviour.
+
+---
+
+# Future Enhancements
+
+These ideas are intentionally outside the scope of the first implementation but may become useful if CCDN grows.
+
+* Delta manifests for large libraries.
+* Geographic origin selection.
+* Optional peer-to-peer mirror synchronisation.
+* Compression for text-based metadata.
+* Signed content release tags.
+* Read-only content mounts between updates.
+
+---
+
+# Deliberately Out of Scope
+
+The following technologies are intentionally excluded from the CCDN design:
+
+* Kubernetes
+* Docker Swarm
+* Redis
+* PostgreSQL
+* Elasticsearch
+* Message queues
+* Dynamic service discovery
+* Distributed databases
+* Complex orchestration systems
+
+The goal is to keep CCDN easy to understand, easy to operate, and easy to recover by relying on mature Unix tooling rather than additional infrastructure.
@@ -0,0 +1,213 @@
+# Community Content Delivery Network Docker nodes
+
+## Background
+
+We wish to deploy a Community CDN that is easy to deploy and maintain.
+
+This can be achieved by providing a Community Managed docker nodes that is regularly updated.
+
+Changes can be applied to the GitTea, and the mirror nodes should pick up the changes.
+
+## Removal of node
+
+- Remove from central DNS
+- Remove from monitoring
+- Remove from allow lists
+- Remove from authorized_keys
+
+## 
+
+- Addition of new nodes
+- Removal of old nodes
+
+
+
+
+Path Validation Strategy
+
+This is where your design differs from a normal web server.
+
+You know:
+
+episode numbers
+valid extensions
+exact file list
+
+Therefore every request can be validated.
+
+Option 1 (recommended)
+
+Allow nginx to serve files normally.
+
+If file does not exist:
+
+error_page 404 = @invalid_request;
+
+location @invalid_request {
+    access_log /var/log/nginx/invalid-paths.log;
+    return 404;
+}
+
+This scales well even with tens of thousands of files.
+
+Prevent Directory Browsing
+
+autoindex off;
+
+location / {
+    try_files $uri =404;
+}
+
+No listings.
+
+No traversal.
+
+No guessing directories.
+
+
+
+
+
+
+
+thank you - I have refined the requirements now so ignore what I said earlier.
+
+We are building a private community content delivery network, with nodes that serves files that typically change once a day.
+
+## Hardware Requirements
+
+Requirements for Hosting
+
+- 24/7 Home Service
+- fixed IP address
+- unlimited bandwidth
+- fast > 500mb/sec upload
+- large > 5T of storage
+- permission from your ISP to run a web server
+- Contact information know to the Janitors
+- Optional: UPS
+
+## Software updates
+
+
+The nodes should ideally run on both [Docker](https://en.wikipedia.org/wiki/Docker_%28software%29) and [Podman](https://en.wikipedia.org/wiki/Podman)
+
+It should run Debian `debian:stable-slim` 
+
+All software MUST be running verifiable Free Libre Open Source software with links to the license.
+
+The nodes should have as little utilities as possible to do it's job and in order to reduce the attack surface.
+
+The amount of traffic is about 5Mb/sec with a load of 2 TPS
+
+There will be many nodes, that will come and go over time.
+
+The nodes will be added to DNS so the load can be shared.
+
+The origin server requirements will be dictated by the needs of the nodes.
+
+Nodes will monitor a rss control channel from the origin server for instructions.
+
+The RSS file will contain a link to a json file.
+
+The json file will contain 
+
+- the fail2ban settings, list of ip addresses, maxretry, findtime and bantime
+- force full rsync Boolean flag - false
+- hours between rsync eg every 3 hours
+- list of origin servers
+- list of Admin IP addresses
+- list of IP addresses to ban immediately 
+- list of useragents to ban immediately 
+
+If the origin server is not available then the nodes should connect to the next origin server in the list.
+
+nodes will synchronize the files from a origin server using rsync over ssh.
+
+A full rsync is sufficient to ensure files are copied correctly.
+
+A full rsync will be done several times a day, eg every 3 hours - from settings
+
+atomic updates are not required as only new files are added.
+
+There are thousands of files
+
+The files will come from a well known directory eg `public_html/eps/hpr${ep_num}/hpr${ep_num}*.${extension}`
+
+The `ep_num` will be a digit from 0001 to 9999
+
+The `extension` will be from a well know list from the origin updated perhaps once a year
+
+nodes will then serve the files using `nginx`
+
+`nginx` must restrict access to files outside the well known files
+
+A list of well known files that clients require (favorite icon, robots.txt, etc) will be available from the origin server.
+
+The files we wish to serve are all Creative Commons Licensed so can be shared.
+
+Random browsing is not allowed.
+
+All file paths are known, and are provided by the origin server.
+
+There should never be any request for a file outside the well known paths.
+
+Any attempt to access any file outside the known paths should be logged.
+
+Based on a configuration provided by the origin server, the server will run failtoban
+
+repeat offenders will be banned for 7 days.
+
+The Admin IP addresses should never be banned
+
+traffic should be secured with let's encrypt of equivalent tls transport
+
+clients can request the files without tls
+
+The nodes public ip address will be placed on a allow list for ssh
+
+Monitoring should be available using Prometheus or similar.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+                    +----------------+
+                    | Origin Server A|
+                    +--------+-------+
+                             |
+                    RSS Control Feed
+                             |
+                    JSON Configuration
+                             |
+          +------------------+------------------+
+          |                                     |
+          v                                     v
+
+----------------+                   +----------------+
+| Edge Node 1    |                   | Edge Node 2    |
+|                |                   |                |
+| nginx          |                   | nginx          |
+| rsync          |                   | rsync          |
+| fail2ban       |                   | fail2ban       |
+| node-agent     |                   | node-agent     |
+| prometheus exp.|                   | prometheus exp.|
+--------+-------+                   +--------+-------+
+         |                                     |
+         +----------------+--------------------+
+                          |
+                        DNS RR
+                          |
+                       Clients
+
+
@@ -0,0 +1,2 @@
+untrusted comment: minisign public key 85F68B90C4D786B9
+RWS5htfEkIv2hSLt8UJmbUF4MV5P0JX+IcHq354RNF6wgC/bO/M3OxqU