From b2bec5b3ee930cd896e0eeb01ce53ecd1aef8ed7 Mon Sep 17 00:00:00 2001 From: jknapp Date: Wed, 21 Oct 2026 12:29:54 -0700 Subject: [PATCH 1/3] feat(build): opt-in PKG_CACHE build arg for the internal package cache Adds scripts/pkg-cache-repos.sh and wires it into Dockerfile, Dockerfile.fpm, Dockerfile.shared-httpd and Dockerfile.shared-ols. With --build-arg PKG_CACHE=http://192.168.1.40 (CT130 pkg-cache), dnf/apt go through the LAN cache. That cache fetches from the Remi/EPEL/AlmaLinux/ Ubuntu/LiteSpeed origins over HTTPS, with fallback tiers and stale-on-error. Without the arg the script is a no-op and builds are unchanged. - rpm: each baseurl becomes "//... /...", so dnf fails over to the origin if the cache dies mid-build - every RUN restores the original repo files before it ends, so no shipped layer carries the LAN address - EPEL/Remi release RPMs come from the cache when it is set - the php.net version gates are untouched Dockerfile.litespeed/.lsphp are not wired yet (see docs/pkg-cache.md). Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 19 ++++-- Dockerfile.fpm | 19 ++++-- Dockerfile.shared-httpd | 13 +++- Dockerfile.shared-ols | 11 ++- docs/pkg-cache.md | 124 +++++++++++++++++++++++++++++++++ scripts/pkg-cache-repos.sh | 136 +++++++++++++++++++++++++++++++++++++ 6 files changed, 311 insertions(+), 11 deletions(-) create mode 100644 docs/pkg-cache.md create mode 100755 scripts/pkg-cache-repos.sh diff --git a/Dockerfile b/Dockerfile index 10046c8..f09612f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,10 +7,15 @@ ARG CACHE_BUST= # php.net's current release for this branch (CI); the build fails on older PHP. ARG EXPECT_PHP_MIN= +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 +# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# are restored at the end of every RUN, so no layer keeps the cache address. +# See scripts/pkg-cache-repos.sh. +ARG PKG_CACHE= +COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh # Install repos, update, install only needed packages, clean up in one layer -RUN dnf install -y \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm \ - https://rpms.remirepo.net/enterprise/remi-release-10.rpm && \ +RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ + dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \ # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out), # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback @@ -18,10 +23,12 @@ RUN dnf install -y \ # replayed (old) http index can't ship old PHP: packages stay GPG-checked and # the EXPECT_PHP_MIN gate below fails the build on anything older than php.net. sed -i -e 's|^#baseurl=http://\(.*\)$|baseurl=https://\1 http://\1|' -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/remi*.repo && \ + /usr/local/sbin/pkg-cache-repos.sh rpm && \ printf 'retries=10\ntimeout=60\n' >> /etc/dnf/dnf.conf && \ dnf update -y && \ dnf install -y httpd mod_ssl openssl wget procps cronie iproute postgresql-devel microdnf less git \ nano rsync unzip zip mariadb bind-utils jq patch nc tree dos2unix && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ dnf clean all && \ rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/* @@ -30,13 +37,17 @@ COPY ./scripts/ /scripts/ RUN chmod +x /scripts/* # Install ImageMagick from EPEL with HEIC/HEIF/AVIF support -RUN dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \ +RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ + dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ dnf clean all # Generate self-signed cert, create needed dirs, install PHP, clean up RUN openssl req -newkey rsa:2048 -nodes -keyout /etc/pki/tls/private/localhost.key -x509 -days 3650 -subj "/CN=localhost" -out /etc/pki/tls/certs/localhost.crt && \ mkdir -p /run/php-fpm/ && \ + /usr/local/sbin/pkg-cache-repos.sh rpm && \ /scripts/install-php$PHPVER.sh && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ # CI passes php.net's current release for the branch; never ship older PHP. if [ -n "$EXPECT_PHP_MIN" ]; then \ php -r 'exit(version_compare(PHP_VERSION, getenv("EXPECT_PHP_MIN"), ">=") ? 0 : 1);' || \ diff --git a/Dockerfile.fpm b/Dockerfile.fpm index 3733d3f..cdcec73 100644 --- a/Dockerfile.fpm +++ b/Dockerfile.fpm @@ -7,10 +7,15 @@ ARG CACHE_BUST= # php.net's current release for this branch (CI); the build fails on older PHP. ARG EXPECT_PHP_MIN= +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 +# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# are restored at the end of every RUN, so no layer keeps the cache address. +# See scripts/pkg-cache-repos.sh. +ARG PKG_CACHE= +COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh # Install repos, update, install only needed packages (no httpd/mod_ssl), clean up in one layer -RUN dnf install -y \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm \ - https://rpms.remirepo.net/enterprise/remi-release-10.rpm && \ +RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ + dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \ # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out), # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback @@ -18,10 +23,12 @@ RUN dnf install -y \ # replayed (old) http index can't ship old PHP: packages stay GPG-checked and # the EXPECT_PHP_MIN gate below fails the build on anything older than php.net. sed -i -e 's|^#baseurl=http://\(.*\)$|baseurl=https://\1 http://\1|' -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/remi*.repo && \ + /usr/local/sbin/pkg-cache-repos.sh rpm && \ printf 'retries=10\ntimeout=60\n' >> /etc/dnf/dnf.conf && \ dnf update -y && \ dnf install -y openssl wget procps cronie iproute postgresql-devel microdnf less git \ nano rsync unzip zip mariadb bind-utils jq patch nc tree dos2unix fcgi && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ dnf clean all && \ rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/* @@ -30,12 +37,16 @@ COPY ./scripts/ /scripts/ RUN chmod +x /scripts/* # Install ImageMagick from EPEL with HEIC/HEIF/AVIF support -RUN dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \ +RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ + dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ dnf clean all # Create needed dirs, install PHP, clean up (no SSL cert, no httpd) RUN mkdir -p /run/php-fpm/ && \ + /usr/local/sbin/pkg-cache-repos.sh rpm && \ /scripts/install-php$PHPVER.sh && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ # CI passes php.net's current release for the branch; never ship older PHP. if [ -n "$EXPECT_PHP_MIN" ]; then \ php -r 'exit(version_compare(PHP_VERSION, getenv("EXPECT_PHP_MIN"), ">=") ? 0 : 1);' || \ diff --git a/Dockerfile.shared-httpd b/Dockerfile.shared-httpd index 5e0c157..b9f36c2 100644 --- a/Dockerfile.shared-httpd +++ b/Dockerfile.shared-httpd @@ -1,10 +1,19 @@ FROM almalinux/10-base +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 +# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# are restored at the end of every RUN, so no layer keeps the cache address. +# See scripts/pkg-cache-repos.sh. +ARG PKG_CACHE= +COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh + # Install Apache and minimal dependencies (no PHP at all) -RUN dnf install -y \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm && \ +RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ + dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls | cut -d' ' -f1) && \ + /usr/local/sbin/pkg-cache-repos.sh rpm && \ dnf update -y && \ dnf install -y httpd mod_ssl openssl iproute cronie procps curl && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ dnf clean all && \ rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/* diff --git a/Dockerfile.shared-ols b/Dockerfile.shared-ols index fea15f6..36787b5 100644 --- a/Dockerfile.shared-ols +++ b/Dockerfile.shared-ols @@ -19,6 +19,13 @@ ARG OLS_VERSION=1.8.4 ARG PHPVER=83 FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER} +## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 +## (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +## are restored at the end of every RUN, so no layer keeps the cache address. +## See scripts/pkg-cache-repos.sh. +ARG PKG_CACHE= +COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh + ## Tooling the shared tier needs on top of the base: ## - inotify-tools: the .htaccess watcher (spec 5.3) ## - gettext-base: envsubst for render-shared-ols-config.sh @@ -28,9 +35,11 @@ FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER} ## liveness check depends on. Only transitively present via the base image ## today (Ubuntu 24.04 pulls it in) — pin it explicitly so it can't be ## pruned as "unused" and silently break the supervisor's crash detection. -RUN apt-get update && \ +RUN /usr/local/sbin/pkg-cache-repos.sh apt && \ + apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ inotify-tools gettext-base openssl ca-certificates curl procps && \ + /usr/local/sbin/pkg-cache-repos.sh restore && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* diff --git a/docs/pkg-cache.md b/docs/pkg-cache.md new file mode 100644 index 0000000..d3171c5 --- /dev/null +++ b/docs/pkg-cache.md @@ -0,0 +1,124 @@ +# Internal package + image cache (`PKG_CACHE`) + +Opt-in build support for the LAN package cache **CT130 `pkg-cache` = `192.168.1.40`** +(Proxmox node `pve`, storage `shared-storage-tn`). Without `PKG_CACHE`, builds use +the upstream repositories exactly as before. + +Why it exists: during the 2026-10-21 PHP security release, builds failed because +Remi mirrors lagged hours behind origin, Remi's origin reset or timed out HTTPS +under load, and an EPEL mirror had a broken certificate. + +## What the cache serves + +| URL on the cache | Upstream tiers, tried in order (stale copy last) | +|---|---| +| `http://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) | +| `http://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` | +| `http://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` | +| `http://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` | +| `http://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` | +| `http://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` | +| `192.168.1.40:5000` (plain http) | Docker Hub pull-through cache (`registry-1.docker.io`) | + +Cache rules: +- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata here means `repomd.xml*`, repodata files whose names don't start with a hash, apt `dists/` except `by-hash/`, `*-latest*.rpm`, `remi-release-N.rpm`, `RPM-GPG-KEY*` and directory listings. +- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are kept for 30 days, and evicted after 60 days without use. The cache is capped at 110 GB (`min_free` 15 GB). +- On a connect error, timeout or 5xx, a request falls through to the next tier. Only the **last** tier answers from a stale copy, so metadata goes primary → fallback → stale. A fresher fallback always wins over stale. +- `proxy_cache_lock` is on: concurrent misses on one file make a single upstream fetch. +- Each response carries `X-Cache-Status` (`HIT`/`MISS`/`REVALIDATED`/`EXPIRED`/`STALE`/`BYPASS`) and `X-Pkgcache-Tier`. On a HIT, the tier header names the location, not the upstream that filled the entry. + +## Building with the cache + +```bash +docker buildx build --build-arg PKG_CACHE=http://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile . +``` + +`scripts/pkg-cache-repos.sh` (copied to `/usr/local/sbin/`) is wired into +`Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`: + +- `rpm`: rewrites the AlmaLinux, EPEL and Remi `baseurl`s to `//... /...`, and comments out that section's mirrorlist/metalink. dnf treats the second URL as a failover, so a dead cache falls back to the origin by itself. +- `apt`: rewrites the Ubuntu archive/security and LiteSpeed URIs to the cache. apt has no failover, so CI must probe the cache first (see below). +- `release-urls`: the EPEL and Remi release RPM URLs, from the cache when it is set. +- `restore`: puts the original repo files back. Every RUN that rewrites ends with `restore`, so **no shipped layer contains the cache address**. Images run on datacenter hosts that cannot reach it. + +`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Their +multi-stage `lsphp-apt-upgrade.sh` flow needs two changes first: +- a `PKG_CACHE` ARG in each stage; +- the madison check accepting the cache URL. It currently requires `*rpms.litespeedtech.com*`, and through the cache `apt-cache madison` shows `http://192.168.1.40/litespeed/debian`. + +The PHP gates are unchanged and still apply: `EXPECT_PHP_MIN`, +`EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`. + +## Why plain HTTP on the LAN (trade-off) + +- The cache fetches upstream over **HTTPS with certificate verification**. Remi and LiteSpeed also have a plain-http origin fallback, which mirrors what the Dockerfile already did. +- Client to cache is plain HTTP. dnf still verifies **package GPG signatures**, and apt verifies the signed `InRelease`/`Release.gpg` plus package hashes. A LAN attacker therefore cannot inject packages. On 2026-10-21 apt measurably refused the cached LiteSpeed index when given the wrong key (`NO_PUBKEY 011AA62DEDA1F085`). +- The residual risk is **metadata replay or freeze**: an attacker on the LAN path, or on the plain-http fallback leg, serves an *old* but validly signed `repomd.xml`. That would let a build install old PHP. Remi/EPEL/AlmaLinux do not enable `repo_gpgcheck`, so dnf does not check metadata signatures. The **php.net version gates** cover this for our images: a build fails if it gets PHP older than php.net's current release. +- WHP **hosts** have no such gate. They should keep the origin as the second `baseurl` and not rely on the cache for freshness on security-release day. Run `pkgcache-refresh` on the cache instead. +- Signing keys must **not** be fetched through the cache. Take them from the base image or directly over HTTPS. +- HTTPS to the cache (internal CA or a Let's Encrypt DNS-01 certificate for `pkg-cache.internal.cloud-hosting.io`) would close the LAN leg. It needs a CA in every build image or a public DNS name plus certificate. Not done; Josh to decide. + +## CI integration plan (NOT applied — needs Josh) + +1. **Network (blocker).** The ephemeral runners (`eph-pve-*` on `pve` vmbr2 10.99.98.0/24, `eph-pve3-*` on `pve3` vmbr1 10.99.99.0/24) are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40. A narrow exception is needed on each node, placed before the DROP lines in `/etc/network/interfaces.d/*`: + ``` + # pve (/etc/network/interfaces.d/ vmbr2 stanza), before the 192.168.0.0/16 DROP: + post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT + post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT + # live, without ifreload: + iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT + # pve3: same with vmbr1 + ``` + `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach 192.168.1.40 is untested. +2. **Probe, then opt in** (no `${{ }}` inside `run:`, per repo rule): + ```yaml + - name: Probe package cache + id: pkgcache + run: | + if curl -fsS --max-time 5 -o /dev/null http://192.168.1.40/ ; then + echo "url=http://192.168.1.40" >> "$GITHUB_OUTPUT" + else + echo "url=" >> "$GITHUB_OUTPUT"; echo "::warning::pkg-cache unreachable - building from upstream" + fi + # in docker/build-push-action `build-args:` add + # PKG_CACHE=${{ steps.pkgcache.outputs.url }} + ``` +3. **Docker Hub mirror for BuildKit.** The workflows use `docker/setup-buildx-action` (docker-container driver). BuildKit **ignores dockerd's `registry-mirrors`**, so configure the builder instead: + ```yaml + - uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + http = true + ``` + This exact config was used for the test builds on 192.168.1.193. Only Docker Hub images go through it; `repo.anhonesthost.net/...` is unaffected. +4. **dockerd clients** (plain `docker pull`, e.g. the build server or test hosts). Untested, because applying it needs a daemon restart. Add to `/etc/docker/daemon.json` and restart docker: + ```json + { "registry-mirrors": ["http://192.168.1.40:5000"], "insecure-registries": ["192.168.1.40:5000"] } + ``` + +## WHP hosts (plan only) + +The production hosts (whp01/whp02/sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts such as 192.168.1.148 can. Options for Josh: +- (a) a second identical cache CT in the datacenter; +- (b) expose this one over NetBird; +- (c) LAN hosts only. + +For a host that can reach a cache, use the same idea as `pkg-cache-repos.sh rpm`, keeping the origin second so the cache is never a single point of failure: + +```ini +# /etc/yum.repos.d/remi-safe.repo [remi-safe] (AL9: enterprise/9) +baseurl=http:///remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/ +#mirrorlist=... +``` + +## Operating the cache (CT130) + +- Config: `/etc/nginx/conf.d/pkgcache-http.conf` (cache zone, maps, log format) and `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf`. `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`: edit its `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`. +- Registry: `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`), service `docker-registry`. +- Data: nginx in `/srv/cache/nginx`, registry in `/srv/cache/registry` (150 GB mount point `mp0`, `backup=0`). +- **Security-release day:** run `pkgcache-refresh` to re-fetch every cached metadata entry now (or `pkgcache-refresh remi` for one prefix; `--list` shows them; `--purge` deletes them instead). A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Without it, metadata is at most 3 minutes old anyway. +- Monitoring: `pkgcache-stats` shows disk, hit ratio, fallback-tier use, STALE count and upstream errors. Logs are in `/var/log/nginx/pkgcache.access.log` (with `cache=`, `tier=`, `up=` fields) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks: `http://192.168.1.40/` and `http://192.168.1.40:5000/v2/`. +- Docker Hub: anonymous pulls are rate-limited per egress IP, and every client now shares this cache's IP. To raise the limit, add `username`/`password` (a Hub access token) under `proxy:`. diff --git a/scripts/pkg-cache-repos.sh b/scripts/pkg-cache-repos.sh new file mode 100755 index 0000000..b378245 --- /dev/null +++ b/scripts/pkg-cache-repos.sh @@ -0,0 +1,136 @@ +#!/bin/bash +## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal +## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster) +## when PKG_CACHE is set; a no-op otherwise, so builds behave exactly as before +## without it. +## +## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi +## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu + LiteSpeed +## pkg-cache-repos.sh release-urls # prints EPEL+Remi release RPM URLs +## pkg-cache-repos.sh restore # put the original repo files back +## +## Every RUN that rewrites must `restore` before it ends, so no shipped layer +## carries the cache's LAN address in /etc/yum.repos.d or /etc/apt (images run on +## hosts that cannot reach it). restore is a no-op when nothing was rewritten. +## +## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging +## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror +## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with +## origin/mirror fallbacks and stale-on-error), keeps metadata only ~3 minutes, +## and keeps immutable .rpm/.deb files long-term. +## +## Integrity is unchanged: dnf/apt still verify package GPG signatures (and apt +## the signed InRelease/Release.gpg); the build's php.net version gates +## (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail a build that gets old PHP. +## +## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if +## the cache dies mid-build dnf fails over to the origin by itself. +## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the +## cache, and a build without PKG_CACHE uses the upstream sources. +set -euo pipefail + +MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-urls|restore}" +STATE=/var/lib/pkg-cache-repos # pristine copies of every file we rewrite + +if [ "$MODE" = restore ]; then + [ -d "$STATE" ] || exit 0 + (cd "$STATE" && find . -type f) | while read -r rel; do + cat "$STATE/$rel" > "/${rel#./}" + done + rm -rf "$STATE" + echo "pkg-cache-repos: original repository files restored" + exit 0 +fi + +# Keep the first-seen (pre-rewrite) copy of a file; later calls in the same RUN +# never overwrite it. +save() { + [ -e "$STATE$1" ] && return 0 + mkdir -p "$STATE$(dirname "$1")" + cp -p "$1" "$STATE$1" +} +C="${PKG_CACHE:-}" +C="${C%/}" + +if [ "$MODE" = release-urls ]; then + if [ -n "$C" ]; then + echo "$C/epel/epel-release-latest-10.noarch.rpm $C/remi/enterprise/remi-release-10.rpm" + else + echo "https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm https://rpms.remirepo.net/enterprise/remi-release-10.rpm" + fi + exit 0 +fi + +if [ -z "$C" ]; then + echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories" + exit 0 +fi +case "$C" in + http://*|https://*) ;; + *) echo "FATAL(pkg-cache-repos): PKG_CACHE must be an http(s) URL, got '$C'" >&2; exit 1 ;; +esac + +rewrite_rpm() { + local f="$1" + grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" || return 0 + save "$f" + # Per [section]: a (commented or live) baseurl on a known origin becomes + # "baseurl=// /", and that section's + # mirrorlist/metalink is commented out (dnf prefers them over baseurl). + # Sections without a known origin (vault debug/source, ...) are untouched. + awk -v C="$C" ' + function flush( i) { + for (i = 1; i <= n; i++) { + if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i] + print buf[i] + } + n = 0; hit = 0 + } + function rw(line, re, prefix, origin, p) { + if (match(line, re)) { + p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p) + hit = 1 + return "baseurl=" C "/" prefix "/" p " " origin "/" p + } + return line + } + /^\[/ { flush() } + { + l = $0 + if (l ~ /^#? ?baseurl=/) { + l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux") + l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel") + l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net") + } + buf[++n] = l + } + END { flush() } + ' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache" +} + +rewrite_apt() { + local f="$1" + grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/' "$f" || return 0 + save "$f" + sed -i -E \ + -e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \ + -e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \ + -e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \ + "$f" +} + +case "$MODE" in + rpm) + for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done + echo "pkg-cache-repos(rpm): repos now via $C:" + grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /' + ;; + apt) + for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + [ -f "$f" ] && rewrite_apt "$f" + done + echo "pkg-cache-repos(apt): sources now via $C:" + grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | grep -F "$C/" | sort -u | sed 's/^/ /' + ;; + *) echo "FATAL(pkg-cache-repos): unknown mode '$MODE'" >&2; exit 1 ;; +esac -- 2.52.0 From 18f65c6d8438aad46ee1a582d3f2d8a04c7565a5 Mon Sep 17 00:00:00 2001 From: jknapp Date: Wed, 21 Oct 2026 12:53:24 -0700 Subject: [PATCH 2/3] fix(pkg-cache): HTTPS-only cache, GPG-verified release RPMs, strict restore Security review follow-up for the opt-in PKG_CACHE path: - PKG_CACHE must be https:// (http is refused). The cache's certificate comes from a name-constrained internal CA (keys/pkg-cache/ca.crt, valid only for 192.168.1.40 / pkg-cache.*). The CA is trusted only by the rewritten repos (dnf sslcacert bundle, apt per-host CaInfo) and only for the duration of the RUN. - epel-release / remi-release are no longer installed straight from a URL. release-install downloads them over HTTPS, checks the package name, imports the signing keys committed in keys/pkg-cache/ and requires `rpm -K` "signatures OK", then installs with localpkg_gpgcheck=1. This applies with and without the cache. - Rewritten repos get gpgcheck=1. Remi and AlmaLinux also get repo_gpgcheck=1, since both sign repomd.xml. EPEL does not publish a repomd signature, so it relies on end-to-end HTTPS. - restore also deletes files it created and any new repo/apt file that still references the cache, then fails the build if a reference remains under /etc/yum.repos.d, /etc/dnf or /etc/apt. - docs: trust model, replay risk, proposed openssl/httpd/ImageMagick minimum-version gates, TLS registry mirror (BuildKit ca= / dockerd certs.d, no insecure-registries). Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 8 +- Dockerfile.fpm | 8 +- Dockerfile.shared-httpd | 8 +- Dockerfile.shared-ols | 6 +- docs/pkg-cache.md | 204 ++++++++++++++++++---------- keys/pkg-cache/README | 20 +++ keys/pkg-cache/RPM-GPG-KEY-EPEL-10 | 29 ++++ keys/pkg-cache/RPM-GPG-KEY-remi2024 | 52 +++++++ keys/pkg-cache/ca.crt | 15 ++ scripts/pkg-cache-repos.sh | 195 ++++++++++++++++++-------- 10 files changed, 411 insertions(+), 134 deletions(-) create mode 100644 keys/pkg-cache/README create mode 100644 keys/pkg-cache/RPM-GPG-KEY-EPEL-10 create mode 100644 keys/pkg-cache/RPM-GPG-KEY-remi2024 create mode 100644 keys/pkg-cache/ca.crt diff --git a/Dockerfile b/Dockerfile index f09612f..3901d6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,15 +7,17 @@ ARG CACHE_BUST= # php.net's current release for this branch (CI); the build fails on older PHP. ARG EXPECT_PHP_MIN= -# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 -# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 +# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are +# GPG-verified against keys/pkg-cache/ either way. Repo files # are restored at the end of every RUN, so no layer keeps the cache address. # See scripts/pkg-cache-repos.sh. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh +COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ # Install repos, update, install only needed packages, clean up in one layer RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ - dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \ + /usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \ # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out), # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback diff --git a/Dockerfile.fpm b/Dockerfile.fpm index cdcec73..76efa55 100644 --- a/Dockerfile.fpm +++ b/Dockerfile.fpm @@ -7,15 +7,17 @@ ARG CACHE_BUST= # php.net's current release for this branch (CI); the build fails on older PHP. ARG EXPECT_PHP_MIN= -# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 -# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 +# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are +# GPG-verified against keys/pkg-cache/ either way. Repo files # are restored at the end of every RUN, so no layer keeps the cache address. # See scripts/pkg-cache-repos.sh. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh +COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ # Install repos, update, install only needed packages (no httpd/mod_ssl), clean up in one layer RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ - dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \ + /usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \ # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out), # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback diff --git a/Dockerfile.shared-httpd b/Dockerfile.shared-httpd index b9f36c2..8b4a5d1 100644 --- a/Dockerfile.shared-httpd +++ b/Dockerfile.shared-httpd @@ -1,15 +1,17 @@ FROM almalinux/10-base -# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 -# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 +# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are +# GPG-verified against keys/pkg-cache/ either way. Repo files # are restored at the end of every RUN, so no layer keeps the cache address. # See scripts/pkg-cache-repos.sh. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh +COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ # Install Apache and minimal dependencies (no PHP at all) RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \ - dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls | cut -d' ' -f1) && \ + /usr/local/sbin/pkg-cache-repos.sh release-install epel && \ /usr/local/sbin/pkg-cache-repos.sh rpm && \ dnf update -y && \ dnf install -y httpd mod_ssl openssl iproute cronie procps curl && \ diff --git a/Dockerfile.shared-ols b/Dockerfile.shared-ols index 36787b5..3e617b7 100644 --- a/Dockerfile.shared-ols +++ b/Dockerfile.shared-ols @@ -19,12 +19,14 @@ ARG OLS_VERSION=1.8.4 ARG PHPVER=83 FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER} -## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40 -## (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files +## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40 +## (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are +## GPG-verified against keys/pkg-cache/ either way. Repo files ## are restored at the end of every RUN, so no layer keeps the cache address. ## See scripts/pkg-cache-repos.sh. ARG PKG_CACHE= COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh +COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/ ## Tooling the shared tier needs on top of the base: ## - inotify-tools: the .htaccess watcher (spec 5.3) diff --git a/docs/pkg-cache.md b/docs/pkg-cache.md index d3171c5..f476dcd 100644 --- a/docs/pkg-cache.md +++ b/docs/pkg-cache.md @@ -1,89 +1,128 @@ # Internal package + image cache (`PKG_CACHE`) -Opt-in build support for the LAN package cache **CT130 `pkg-cache` = `192.168.1.40`** -(Proxmox node `pve`, storage `shared-storage-tn`). Without `PKG_CACHE`, builds use -the upstream repositories exactly as before. +Opt-in build support for the LAN package cache **CT130 `pkg-cache` at `https://192.168.1.40`** (Proxmox node `pve`, storage `shared-storage-tn`). -Why it exists: during the 2026-10-21 PHP security release, builds failed because -Remi mirrors lagged hours behind origin, Remi's origin reset or timed out HTTPS -under load, and an EPEL mirror had a broken certificate. +Without `PKG_CACHE`, builds use the upstream repositories. The one change from before is that the EPEL and Remi release RPMs are now **GPG-verified** against `keys/pkg-cache/` before they are installed. + +Why it exists: during the 2026-10-21 PHP security release, builds failed because: +- Remi mirrors lagged hours behind origin; +- Remi's origin reset or timed out HTTPS under load; +- an EPEL mirror had a broken certificate. + +On that day a build without the cache spent **1031 s** in its first dnf step. Through the cache the same step took **40 s cold and 21 s warm**. ## What the cache serves -| URL on the cache | Upstream tiers, tried in order (stale copy last) | +| URL (https, pkg-cache CA) | Upstream tiers, tried in order (stale copy last) | |---|---| -| `http://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) | -| `http://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` | -| `http://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` | -| `http://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` | -| `http://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` | -| `http://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` | -| `192.168.1.40:5000` (plain http) | Docker Hub pull-through cache (`registry-1.docker.io`) | +| `https://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) | +| `https://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` | +| `https://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` | +| `https://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` | +| `https://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` | +| `https://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` | +| `192.168.1.40:5000` (TLS, pkg-cache CA) | Docker Hub pull-through cache (`registry-1.docker.io`) | + +Plain `http://192.168.1.40/` answers **403** to LAN clients. Cache rules: -- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata here means `repomd.xml*`, repodata files whose names don't start with a hash, apt `dists/` except `by-hash/`, `*-latest*.rpm`, `remi-release-N.rpm`, `RPM-GPG-KEY*` and directory listings. -- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are kept for 30 days, and evicted after 60 days without use. The cache is capped at 110 GB (`min_free` 15 GB). -- On a connect error, timeout or 5xx, a request falls through to the next tier. Only the **last** tier answers from a stale copy, so metadata goes primary → fallback → stale. A fresher fallback always wins over stale. -- `proxy_cache_lock` is on: concurrent misses on one file make a single upstream fetch. -- Each response carries `X-Cache-Status` (`HIT`/`MISS`/`REVALIDATED`/`EXPIRED`/`STALE`/`BYPASS`) and `X-Pkgcache-Tier`. On a HIT, the tier header names the location, not the upstream that filled the entry. +- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata means: + - `repomd.xml*` + - repodata files whose names don't start with a hash + - apt `dists/`, except `by-hash/` + - `*-latest*.rpm` and `remi-release-N.rpm` + - `RPM-GPG-KEY*` + - directory listings +- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are cached for 30 days and evicted after 60 days unused. Total size is capped at 110 GB, with 15 GB kept free. +- On a connect error, timeout or 5xx, the request falls through to the next tier. Only the **last** tier may answer from a stale copy, so the order is primary, then fallback, then stale. +- `proxy_cache_lock` is on. +- Responses carry `X-Cache-Status` and `X-Pkgcache-Tier`. ## Building with the cache ```bash -docker buildx build --build-arg PKG_CACHE=http://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile . +docker buildx build --build-arg PKG_CACHE=https://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile . ``` -`scripts/pkg-cache-repos.sh` (copied to `/usr/local/sbin/`) is wired into -`Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`: +`scripts/pkg-cache-repos.sh` (installed as `/usr/local/sbin/`) and `keys/pkg-cache/` (installed as `/usr/local/share/pkg-cache/`) are wired into `Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`. The script has four modes. -- `rpm`: rewrites the AlmaLinux, EPEL and Remi `baseurl`s to `//... /...`, and comments out that section's mirrorlist/metalink. dnf treats the second URL as a failover, so a dead cache falls back to the origin by itself. -- `apt`: rewrites the Ubuntu archive/security and LiteSpeed URIs to the cache. apt has no failover, so CI must probe the cache first (see below). -- `release-urls`: the EPEL and Remi release RPM URLs, from the cache when it is set. -- `restore`: puts the original repo files back. Every RUN that rewrites ends with `restore`, so **no shipped layer contains the cache address**. Images run on datacenter hosts that cannot reach it. +**`rpm`** handles AlmaLinux, EPEL and Remi: +- Each `baseurl` becomes `//... /...`. dnf fails over to the origin if the cache dies. +- That section's mirrorlist/metalink is commented out. +- The section gets `sslcacert=` pointing at a bundle of the system CAs plus the pkg-cache CA. The bundle lives in the state dir and is deleted by `restore`. +- It also gets `gpgcheck=1`. +- AlmaLinux and Remi also get `repo_gpgcheck=1`, because both sign `repomd.xml` (measured 2026-10-21: `repomd.xml.asc` returns 200). EPEL publishes no repomd signature (404), so its metadata integrity rests on HTTPS end to end. -`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Their -multi-stage `lsphp-apt-upgrade.sh` flow needs two changes first: -- a `PKG_CACHE` ARG in each stage; -- the madison check accepting the cache URL. It currently requires `*rpms.litespeedtech.com*`, and through the cache `apt-cache madison` shows `http://192.168.1.40/litespeed/debian`. +**`apt`** handles Ubuntu archive/security and LiteSpeed: +- Their URIs are rewritten to the cache. +- `/etc/apt/apt.conf.d/99pkg-cache` sets `Acquire::https::192.168.1.40::CaInfo`, so the CA is trusted for the cache host only. +- apt verifies the signed `InRelease`/`Release.gpg` as always. It has no failover, so CI must probe the cache first. -The PHP gates are unchanged and still apply: `EXPECT_PHP_MIN`, -`EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`. +**`release-install epel [remi]`**: +- Downloads the release RPM over HTTPS, from the cache or from the origin. +- Checks the package name. +- Imports the key committed in `keys/pkg-cache/`. +- Requires `rpm -K` to report `digests signatures OK`. +- Installs with `--setopt=localpkg_gpgcheck=1`. -## Why plain HTTP on the LAN (trade-off) +**`restore`**: +- Puts rewritten files back. +- Deletes files the script created. +- Deletes any repo/apt file that **appeared during the RUN** and still references the cache. +- Then **fails the build** if any file under `/etc/yum.repos.d`, `/etc/dnf` or `/etc/apt` still mentions the cache. -- The cache fetches upstream over **HTTPS with certificate verification**. Remi and LiteSpeed also have a plain-http origin fallback, which mirrors what the Dockerfile already did. -- Client to cache is plain HTTP. dnf still verifies **package GPG signatures**, and apt verifies the signed `InRelease`/`Release.gpg` plus package hashes. A LAN attacker therefore cannot inject packages. On 2026-10-21 apt measurably refused the cached LiteSpeed index when given the wrong key (`NO_PUBKEY 011AA62DEDA1F085`). -- The residual risk is **metadata replay or freeze**: an attacker on the LAN path, or on the plain-http fallback leg, serves an *old* but validly signed `repomd.xml`. That would let a build install old PHP. Remi/EPEL/AlmaLinux do not enable `repo_gpgcheck`, so dnf does not check metadata signatures. The **php.net version gates** cover this for our images: a build fails if it gets PHP older than php.net's current release. -- WHP **hosts** have no such gate. They should keep the origin as the second `baseurl` and not rely on the cache for freshness on security-release day. Run `pkgcache-refresh` on the cache instead. -- Signing keys must **not** be fetched through the cache. Take them from the base image or directly over HTTPS. -- HTTPS to the cache (internal CA or a Let's Encrypt DNS-01 certificate for `pkg-cache.internal.cloud-hosting.io`) would close the LAN leg. It needs a CA in every build image or a public DNS name plus certificate. Not done; Josh to decide. +Every RUN that rewrites ends with `restore`, so no shipped layer carries the LAN address. This was verified on the built images: 0 references, no state dir, no apt drop-in. + +`PKG_CACHE` must start with `https://`. `http://` is refused. + +`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Two things are needed first: +- the multi-stage `lsphp-apt-upgrade.sh` flow needs a `PKG_CACHE` ARG in each stage; +- its madison check must accept the cache URL. It currently requires `*rpms.litespeedtech.com*`, and via the cache madison shows `https://192.168.1.40/litespeed/debian`. + +## Trust model + +`keys/pkg-cache/ca.crt` is the *pkg-cache internal CA*: +- EC P-256, valid to 2036. +- **Name-constrained** to IP `192.168.1.40` and DNS `pkg-cache.internal.cloud-hosting.io` / `pkg-cache.geekhome.local`. Even its private key, which exists only on CT130 at `/etc/pkgcache-ca/ca.key`, cannot mint a certificate that clients would accept for any other host. +- It is trusted only by the rewritten repos, the apt per-host option and the BuildKit/dockerd registry config. It is never added to a system trust store. + +The server certificate `/etc/pkgcache-ca/server.crt` **expires 2029-10-20**. `pkgcache-stats` prints the expiry. Renew it with the same `openssl x509 -req ... -extfile srv.cnf` steps in `/etc/pkgcache-ca/`. + +Remaining risks: +- **Replay of old but validly signed metadata.** Signatures stop tampering, not freshness. The cache's own plain-http tiers for Remi and LiteSpeed are the leg where an internet MITM could replay. The php.net gates (`EXPECT_PHP_MIN`, `EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`) fail an image build that gets old PHP. **Josh to decide** whether to drop those http tiers: you gain replay resistance and lose availability when Remi's HTTPS is down, as it was all day on 2026-10-21. +- **Proposed, not implemented:** the same kind of minimum-version gate for other security-relevant packages: + - CI resolves the newest `openssl`/`openssl-libs` and `httpd` from AlmaLinux BaseOS/AppStream, and `ImageMagick` from EPEL, **directly from the origin over HTTPS** (e.g. `dnf repoquery --repofrompath=o,https://repo.almalinux.org/almalinux/10/BaseOS/x86_64/os/ --repo=o --latest-limit=1 --qf '%{evr}' openssl-libs`). + - It passes the results as `EXPECT_OPENSSL_MIN` / `EXPECT_HTTPD_MIN` / `EXPECT_IMAGEMAGICK_MIN`. + - The Dockerfile asserts `rpm -q --qf '%{EVR}'` is not older (`rpm --eval "%{lua: print(rpm.vercmp('A','B'))}"`). ## CI integration plan (NOT applied — needs Josh) -1. **Network (blocker).** The ephemeral runners (`eph-pve-*` on `pve` vmbr2 10.99.98.0/24, `eph-pve3-*` on `pve3` vmbr1 10.99.99.0/24) are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40. A narrow exception is needed on each node, placed before the DROP lines in `/etc/network/interfaces.d/*`: +1. **Network (blocker).** The ephemeral runners are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40: + - `eph-pve-*` on `pve`, vmbr2, 10.99.98.0/24; + - `eph-pve3-*` on `pve3`, vmbr1, 10.99.99.0/24. + + A narrow exception is needed, placed before the DROP lines: ``` - # pve (/etc/network/interfaces.d/ vmbr2 stanza), before the 192.168.0.0/16 DROP: - post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT - post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT - # live, without ifreload: - iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT - # pve3: same with vmbr1 + # pve, /etc/network/interfaces.d/ vmbr2 stanza, BEFORE the 192.168.0.0/16 DROP: + post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT + post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT + # live: iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT + # pve3: the same with vmbr1 ``` - `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach 192.168.1.40 is untested. -2. **Probe, then opt in** (no `${{ }}` inside `run:`, per repo rule): + `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach the cache is untested. +2. **Probe with the CA, then opt in.** Keep `${{ }}` out of `run:`. The probe fails when the cache is down **or its certificate is expired or untrusted**, and the build then uses upstream. ```yaml - name: Probe package cache id: pkgcache run: | - if curl -fsS --max-time 5 -o /dev/null http://192.168.1.40/ ; then - echo "url=http://192.168.1.40" >> "$GITHUB_OUTPUT" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/ ; then + echo "url=https://192.168.1.40" >> "$GITHUB_OUTPUT" else echo "url=" >> "$GITHUB_OUTPUT"; echo "::warning::pkg-cache unreachable - building from upstream" fi - # in docker/build-push-action `build-args:` add - # PKG_CACHE=${{ steps.pkgcache.outputs.url }} + # docker/build-push-action `build-args:` add PKG_CACHE=${{ steps.pkgcache.outputs.url }} ``` -3. **Docker Hub mirror for BuildKit.** The workflows use `docker/setup-buildx-action` (docker-container driver). BuildKit **ignores dockerd's `registry-mirrors`**, so configure the builder instead: +3. **Docker Hub mirror for BuildKit**, over TLS and without `insecure`. `setup-buildx-action` uses the docker-container driver, which **ignores dockerd's `registry-mirrors`**. buildx copies the `ca` file into the builder. ```yaml - uses: docker/setup-buildx-action@v3 with: @@ -91,34 +130,61 @@ The PHP gates are unchanged and still apply: `EXPECT_PHP_MIN`, [registry."docker.io"] mirrors = ["192.168.1.40:5000"] [registry."192.168.1.40:5000"] - http = true + ca = ["keys/pkg-cache/ca.crt"] ``` - This exact config was used for the test builds on 192.168.1.193. Only Docker Hub images go through it; `repo.anhonesthost.net/...` is unaffected. -4. **dockerd clients** (plain `docker pull`, e.g. the build server or test hosts). Untested, because applying it needs a daemon restart. Add to `/etc/docker/daemon.json` and restart docker: - ```json - { "registry-mirrors": ["http://192.168.1.40:5000"], "insecure-registries": ["192.168.1.40:5000"] } + The exact equivalent (absolute `ca` path) was measured on 192.168.1.193. BuildKit falls back to Docker Hub when the mirror is down; this was measured too, with the registry stopped. The relative `ca` path inside a workflow is **untested**; use `${{ github.workspace }}/keys/pkg-cache/ca.crt` if it doesn't resolve. +4. **dockerd clients** (plain `docker pull`). Untested, because it needs a daemon restart. Use TLS, no `insecure-registries`: + ``` + install -D -m0644 keys/pkg-cache/ca.crt /etc/docker/certs.d/192.168.1.40:5000/ca.crt + # /etc/docker/daemon.json: { "registry-mirrors": ["https://192.168.1.40:5000"] } then restart docker ``` ## WHP hosts (plan only) -The production hosts (whp01/whp02/sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts such as 192.168.1.148 can. Options for Josh: -- (a) a second identical cache CT in the datacenter; +The production hosts (whp01, whp02, sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts can, e.g. 192.168.1.148. Options for Josh: +- (a) a second identical cache in the datacenter, with its own name-constrained CA; - (b) expose this one over NetBird; - (c) LAN hosts only. -For a host that can reach a cache, use the same idea as `pkg-cache-repos.sh rpm`, keeping the origin second so the cache is never a single point of failure: +For a host that can reach a cache: +- install the CA at `/etc/pki/pkg-cache/ca.crt`; +- create a bundle (`cat /etc/pki/tls/certs/ca-bundle.crt /etc/pki/pkg-cache/ca.crt > /etc/pki/pkg-cache/bundle.pem`); +- in each repo section, keep the origin as the second URL: ```ini -# /etc/yum.repos.d/remi-safe.repo [remi-safe] (AL9: enterprise/9) -baseurl=http:///remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/ +[remi-safe] +baseurl=https:///remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/ #mirrorlist=... +sslcacert=/etc/pki/pkg-cache/bundle.pem +gpgcheck=1 +repo_gpgcheck=1 # Remi + AlmaLinux sign repomd; EPEL does not ``` +Hosts have no php.net gate. On a security-release day, run `pkgcache-refresh` on the cache and check `dnf info php-cli` against php.net. + ## Operating the cache (CT130) -- Config: `/etc/nginx/conf.d/pkgcache-http.conf` (cache zone, maps, log format) and `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf`. `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`: edit its `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`. -- Registry: `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`), service `docker-registry`. -- Data: nginx in `/srv/cache/nginx`, registry in `/srv/cache/registry` (150 GB mount point `mp0`, `backup=0`). -- **Security-release day:** run `pkgcache-refresh` to re-fetch every cached metadata entry now (or `pkgcache-refresh remi` for one prefix; `--list` shows them; `--purge` deletes them instead). A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Without it, metadata is at most 3 minutes old anyway. -- Monitoring: `pkgcache-stats` shows disk, hit ratio, fallback-tier use, STALE count and upstream errors. Logs are in `/var/log/nginx/pkgcache.access.log` (with `cache=`, `tier=`, `up=` fields) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks: `http://192.168.1.40/` and `http://192.168.1.40:5000/v2/`. -- Docker Hub: anonymous pulls are rate-limited per egress IP, and every client now shares this cache's IP. To raise the limit, add `username`/`password` (a Hub access token) under `proxy:`. +- **nginx config.** + - `/etc/nginx/conf.d/pkgcache-http.conf` holds the cache zone, maps and log format. + - `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf` hold the shared settings. + - `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`. Edit the `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`. + - Listeners: :443 TLS for the LAN, `127.0.0.1:80` for local tools, and :80 for the LAN answers 403. +- **Registry.** Config is `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`, with TLS). Service `docker-registry`. +- **TLS.** Everything is in `/etc/pkgcache-ca/`: `ca.key` (0600), `ca.crt`, `server.key`, `server-chain.crt`, and the `ca.cnf`/`srv.cnf` openssl configs. +- **Data.** nginx uses `/srv/cache/nginx`, the registry uses `/srv/cache/registry`. Both live on the 150 GB mount point `mp0` (`backup=0`). +- **Security-release day.** `pkgcache-refresh` re-fetches every cached metadata entry right away. Variants: + - `pkgcache-refresh remi` refreshes one prefix; + - `--list` lists the entries; + - `--purge` deletes them instead. + + A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Metadata is at most 3 minutes old anyway. +- **Monitoring.** `pkgcache-stats` shows: + - disk usage; + - hit ratio; + - fallback-tier use; + - the STALE count; + - upstream errors; + - certificate expiry. + + Logs are `/var/log/nginx/pkgcache.access.log` (fields `cache=`, `tier=`, `up=`) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks are `https://192.168.1.40/` and `https://192.168.1.40:5000/v2/`, with certificate checking against the CA. +- **Docker Hub rate limits.** Anonymous pulls are limited per egress IP, and all clients share it. Add `username`/`password` (a Hub access token) under `proxy:` if needed. diff --git a/keys/pkg-cache/README b/keys/pkg-cache/README new file mode 100644 index 0000000..bdcc04b --- /dev/null +++ b/keys/pkg-cache/README @@ -0,0 +1,20 @@ +Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh, +docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing +here is added to the system trust store. + +ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18. + NAME-CONSTRAINED to IP 192.168.1.40 and DNS + pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local. + SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3 + (private key lives only on CT130: /etc/pkgcache-ca/ca.key) +RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158 + fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and + mirror.math.princeton.edu (identical); fingerprint matches + https://fedoraproject.org/security/ +RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4 + fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical + to the key shipped in remi-release-10.2-1.el10.remi. Not found on + keys.openpgp.org / keyserver.ubuntu.com - confirm out of band. + Remi rotates keys (remi2025, ...): when remi-release starts being + signed by a new key, `release-install remi` fails loudly - add the + new key here (and to pkg-cache-repos.sh). diff --git a/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 b/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 new file mode 100644 index 0000000..adf4f88 --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-EPEL-10 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA +Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK +Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L +URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf +mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2 +XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI +uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60 +GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix +fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z +BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77 +HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB +tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB +CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC +BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p +l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp +yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg +Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u +scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs +7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK +F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg +sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB +0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY +1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt +TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k +p9Dfd2pKuxK9Mys= +=mhQZ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/RPM-GPG-KEY-remi2024 b/keys/pkg-cache/RPM-GPG-KEY-remi2024 new file mode 100644 index 0000000..fe82d62 --- /dev/null +++ b/keys/pkg-cache/RPM-GPG-KEY-remi2024 @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W +Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe +UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa +qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN +ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB +jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO +/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL +yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ +w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN +0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l +ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB +tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly +ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd +/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX +gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx +Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W +4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/ +/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz +/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc +3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof +ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS +dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy +sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n +i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd +DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P +7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI +T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ +OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3 +cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W +RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x +5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m +8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD +pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r +N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm +Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG +Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA +AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9 +OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI +wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA +qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c +WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV +u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI +qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S +4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4 +MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II +Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl +bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s +r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1 +=meaU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/keys/pkg-cache/ca.crt b/keys/pkg-cache/ca.crt new file mode 100644 index 0000000..4413e94 --- /dev/null +++ b/keys/pkg-cache/ca.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw +TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV +BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN +MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg +KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV +OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/ +v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw +a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu +Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg +cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM +YAIl +-----END CERTIFICATE----- diff --git a/scripts/pkg-cache-repos.sh b/scripts/pkg-cache-repos.sh index b378245..fb83c55 100755 --- a/scripts/pkg-cache-repos.sh +++ b/scripts/pkg-cache-repos.sh @@ -1,63 +1,140 @@ #!/bin/bash ## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal ## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster) -## when PKG_CACHE is set; a no-op otherwise, so builds behave exactly as before -## without it. +## when PKG_CACHE is set; without it the repo files are left alone and builds +## use the upstream repositories as before. ## -## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi -## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu + LiteSpeed -## pkg-cache-repos.sh release-urls # prints EPEL+Remi release RPM URLs -## pkg-cache-repos.sh restore # put the original repo files back +## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi +## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu + LiteSpeed +## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs +## pkg-cache-repos.sh restore # undo everything above ## -## Every RUN that rewrites must `restore` before it ends, so no shipped layer -## carries the cache's LAN address in /etc/yum.repos.d or /etc/apt (images run on -## hosts that cannot reach it). restore is a no-op when nothing was rewritten. +## Every RUN that rewrites must `restore` before it ends: it puts rewritten files +## back, deletes files this script created, deletes any NEW repo/apt file that +## still references the cache, then FAILS if any reference is left — so no shipped +## layer carries the cache's LAN address (images run on hosts that can't reach it). ## ## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging ## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror ## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with -## origin/mirror fallbacks and stale-on-error), keeps metadata only ~3 minutes, -## and keeps immutable .rpm/.deb files long-term. +## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and +## immutable .rpm/.deb files long-term. ## -## Integrity is unchanged: dnf/apt still verify package GPG signatures (and apt -## the signed InRelease/Release.gpg); the build's php.net version gates -## (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail a build that gets old PHP. +## Integrity (security review 2026-10-21): +## * PKG_CACHE must be https://. The cache's certificate is issued by the +## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only +## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten +## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN — +## never added to the system trust store. +## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and +## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL +## publishes no repomd signature (404), so its metadata relies on HTTPS end +## to end (client->cache->dl.fedoraproject.org/mirrors, all verified). +## * release-install downloads epel-release / remi-release (from the cache, or +## from the HTTPS origin without PKG_CACHE), checks the package name, imports +## the signing keys committed in keys/pkg-cache/ and requires +## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1. +## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail +## a build that gets old PHP (replay of old-but-signed metadata). ## ## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if ## the cache dies mid-build dnf fails over to the origin by itself. ## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the -## cache, and a build without PKG_CACHE uses the upstream sources. +## cache (with the CA), and a build without PKG_CACHE uses upstream sources. set -euo pipefail -MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-urls|restore}" -STATE=/var/lib/pkg-cache-repos # pristine copies of every file we rewrite - -if [ "$MODE" = restore ]; then - [ -d "$STATE" ] || exit 0 - (cd "$STATE" && find . -type f) | while read -r rel; do - cat "$STATE/$rel" > "/${rel#./}" - done - rm -rf "$STATE" - echo "pkg-cache-repos: original repository files restored" - exit 0 -fi - -# Keep the first-seen (pre-rewrite) copy of a file; later calls in the same RUN -# never overwrite it. -save() { - [ -e "$STATE$1" ] && return 0 - mkdir -p "$STATE$(dirname "$1")" - cp -p "$1" "$STATE$1" -} +MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|restore}" +shift || true +STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files +SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-* C="${PKG_CACHE:-}" C="${C%/}" -if [ "$MODE" = release-urls ]; then +die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; } + +if [ -n "$C" ]; then + case "$C" in + https://*) ;; + *) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;; + esac + HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}" + [ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing" +fi + +# Files that may hold repo configuration; restore re-checks all of them. +repo_files() { + find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true +} + +start_session() { + [ -d "$STATE" ] && return 0 + mkdir -p "$STATE/orig" + repo_files > "$STATE/manifest" # what existed before this RUN touched anything + : > "$STATE/created" + printf '%s\n' "$HOSTPORT" > "$STATE/host" +} +save() { # keep the first-seen (pre-rewrite) copy + [ -e "$STATE/orig$1" ] && return 0 + mkdir -p "$STATE/orig$(dirname "$1")" + cp -p "$1" "$STATE/orig$1" +} +created() { printf '%s\n' "$1" >> "$STATE/created"; } + +if [ "$MODE" = restore ]; then + [ -d "$STATE" ] || exit 0 + host=$(cat "$STATE/host") + (cd "$STATE/orig" && find . -type f) | while read -r rel; do + cat "$STATE/orig/$rel" > "/${rel#./}" + done + while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created" + # A repo/apt file that appeared during this RUN and still names the cache + # (e.g. written by a package from the rewritten repos) is removed too. + repo_files | while read -r f; do + if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then + echo "pkg-cache-repos: removing new file that references the cache: $f" + rm -f -- "$f" + fi + done + left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true) + rm -rf "$STATE" + [ -z "$left" ] || die "cache address $host still present after restore in: $left" + echo "pkg-cache-repos: original repository files restored (no reference to $host left)" + exit 0 +fi + +if [ "$MODE" = release-install ]; then + [ $# -gt 0 ] || die "release-install needs: epel and/or remi" + tmp=$(mktemp -d) + curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120) if [ -n "$C" ]; then - echo "$C/epel/epel-release-latest-10.noarch.rpm $C/remi/enterprise/remi-release-10.rpm" - else - echo "https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm https://rpms.remirepo.net/enterprise/remi-release-10.rpm" + start_session + cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem" + curl_opts+=(--cacert "$STATE/ca-bundle.pem") fi + files=() + for name in "$@"; do + case "$name" in + epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-10 + path=/epel/epel-release-latest-10.noarch.rpm + origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm ;; + remi) pkg=remi-release key=RPM-GPG-KEY-remi2024 + path=/remi/enterprise/remi-release-10.rpm + origin=https://rpms.remirepo.net/enterprise/remi-release-10.rpm ;; + *) die "unknown release package '$name'" ;; + esac + url="$origin"; [ -n "$C" ] && url="$C$path" + f="$tmp/$pkg.rpm" + curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url" + rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key" + got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true) + [ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg" + chk=$(rpm -K "$f" 2>&1 || true) + [[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk" + echo "pkg-cache-repos: $pkg verified ($chk) from $url" + files+=("$f") + done + dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}" + rm -rf "$tmp" exit 0 fi @@ -65,31 +142,34 @@ if [ -z "$C" ]; then echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories" exit 0 fi -case "$C" in - http://*|https://*) ;; - *) echo "FATAL(pkg-cache-repos): PKG_CACHE must be an http(s) URL, got '$C'" >&2; exit 1 ;; -esac rewrite_rpm() { local f="$1" grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" || return 0 save "$f" # Per [section]: a (commented or live) baseurl on a known origin becomes - # "baseurl=// /", and that section's - # mirrorlist/metalink is commented out (dnf prefers them over baseurl). - # Sections without a known origin (vault debug/source, ...) are untouched. - awk -v C="$C" ' + # "baseurl=// /"; that section's + # mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it + # gets sslcacert=, gpgcheck=1 and, for repos + # that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched. + awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" ' function flush( i) { for (i = 1; i <= n; i++) { if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i] + if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue print buf[i] } - n = 0; hit = 0 + if (hit) { + print "sslcacert=" BUNDLE + print "gpgcheck=1" + if (signed) print "repo_gpgcheck=1" + } + n = 0; hit = 0; signed = 0 } - function rw(line, re, prefix, origin, p) { + function rw(line, re, prefix, origin, sig, p) { if (match(line, re)) { p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p) - hit = 1 + hit = 1; if (sig) signed = 1 return "baseurl=" C "/" prefix "/" p " " origin "/" p } return line @@ -98,9 +178,9 @@ rewrite_rpm() { { l = $0 if (l ~ /^#? ?baseurl=/) { - l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux") - l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel") - l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net") + l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1) + l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0) + l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1) } buf[++n] = l } @@ -121,16 +201,23 @@ rewrite_apt() { case "$MODE" in rpm) + start_session + cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem" for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done echo "pkg-cache-repos(rpm): repos now via $C:" grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /' ;; apt) + start_session + conf=/etc/apt/apt.conf.d/99pkg-cache + # Trust the pkg-cache CA for the cache host ONLY (apt per-host option). + printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf" + created "$conf" for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do [ -f "$f" ] && rewrite_apt "$f" done echo "pkg-cache-repos(apt): sources now via $C:" grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | grep -F "$C/" | sort -u | sed 's/^/ /' ;; - *) echo "FATAL(pkg-cache-repos): unknown mode '$MODE'" >&2; exit 1 ;; + *) die "unknown mode '$MODE'" ;; esac -- 2.52.0 From 2b9e7f302cb021987c77cab50ec7fe81dad34f77 Mon Sep 17 00:00:00 2001 From: jknapp Date: Wed, 21 Oct 2026 16:46:03 -0700 Subject: [PATCH 3/3] ci: enable PKG_CACHE via a TLS probe that falls back to upstream; host repo tool - build-push.yaml: Build-and-Push, Build-FPM-Images, Build-Shared-httpd and Build-Shared-OLS probe https://192.168.1.40 and :5000 with the pinned pkg-cache CA (5 s timeouts). When the cache answers they pass PKG_CACHE and configure the BuildKit Docker Hub mirror (ca=). When it doesn't, they build from upstream exactly as before and emit a warning. Fallback measured from the runner bridge: refused <1 s, silent drop 10 s. - scripts/host/pkgcache-host-repos.sh: apply/revert/status for WHP hosts. Each rewritten repo uses baseurl= as the real fallback; mirrorlist/metalink stay commented because measured dnf ignores baseurl when they are present. Keeps gpgcheck=1 / repo_gpgcheck=1 and backs up every file. Tested on 192.168.1.148, including cache-down and silent-drop fallback. - docs: runner firewall rule (applied), measured fallback results. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build-push.yaml | 125 ++++++++++++++++++++++++++++ docs/pkg-cache.md | 120 ++++++++++++++------------ scripts/host/pkgcache-host-repos.sh | 107 ++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 51 deletions(-) create mode 100644 scripts/host/pkgcache-host-repos.sh diff --git a/.gitea/workflows/build-push.yaml b/.gitea/workflows/build-push.yaml index e6d8a20..bbca7d2 100644 --- a/.gitea/workflows/build-push.yaml +++ b/.gitea/workflows/build-push.yaml @@ -85,10 +85,40 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if + # it answers over TLS with our pinned CA; otherwise (down, unreachable, + # bad/expired cert) this job builds from the upstream repos and Docker Hub + # exactly as before. The repo cache and the registry mirror are probed + # separately so either can fail back on its own. + - name: Probe package cache + id: pkgcache + run: | + url=""; mirror="" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi + echo "pkg-cache: repos=${url:-} registry-mirror=${mirror:-}" + [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "mirror=${mirror}" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + # BuildKit (docker-container driver) ignores dockerd's registry-mirrors, + # so the mirror is configured on the builder; BuildKit itself still falls + # back to Docker Hub if the mirror stops answering mid-run. + - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror) + if: steps.pkgcache.outputs.mirror != '' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"] + - name: Set up Docker Buildx + if: steps.pkgcache.outputs.mirror == '' uses: docker/setup-buildx-action@v3 # Per-run cache-bust value. Each job's buildx builder is created fresh by @@ -135,6 +165,7 @@ jobs: build-args: | PHPVER=${{ matrix.phpver }} CACHE_BUST=${{ steps.bust.outputs.value }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} tags: | repo.anhonesthost.net/cloud-hosting-platform/cac:php${{ matrix.phpver }} @@ -152,10 +183,40 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if + # it answers over TLS with our pinned CA; otherwise (down, unreachable, + # bad/expired cert) this job builds from the upstream repos and Docker Hub + # exactly as before. The repo cache and the registry mirror are probed + # separately so either can fail back on its own. + - name: Probe package cache + id: pkgcache + run: | + url=""; mirror="" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi + echo "pkg-cache: repos=${url:-} registry-mirror=${mirror:-}" + [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "mirror=${mirror}" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + # BuildKit (docker-container driver) ignores dockerd's registry-mirrors, + # so the mirror is configured on the builder; BuildKit itself still falls + # back to Docker Hub if the mirror stops answering mid-run. + - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror) + if: steps.pkgcache.outputs.mirror != '' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"] + - name: Set up Docker Buildx + if: steps.pkgcache.outputs.mirror == '' uses: docker/setup-buildx-action@v3 # Per-run cache-bust value. Each job's buildx builder is created fresh by @@ -203,6 +264,7 @@ jobs: build-args: | PHPVER=${{ matrix.phpver }} CACHE_BUST=${{ steps.bust.outputs.value }} + PKG_CACHE=${{ steps.pkgcache.outputs.url }} EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }} tags: | repo.anhonesthost.net/cloud-hosting-platform/cac-fpm:php${{ matrix.phpver }} @@ -417,10 +479,40 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if + # it answers over TLS with our pinned CA; otherwise (down, unreachable, + # bad/expired cert) this job builds from the upstream repos and Docker Hub + # exactly as before. The repo cache and the registry mirror are probed + # separately so either can fail back on its own. + - name: Probe package cache + id: pkgcache + run: | + url=""; mirror="" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi + echo "pkg-cache: repos=${url:-} registry-mirror=${mirror:-}" + [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "mirror=${mirror}" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + # BuildKit (docker-container driver) ignores dockerd's registry-mirrors, + # so the mirror is configured on the builder; BuildKit itself still falls + # back to Docker Hub if the mirror stops answering mid-run. + - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror) + if: steps.pkgcache.outputs.mirror != '' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"] + - name: Set up Docker Buildx + if: steps.pkgcache.outputs.mirror == '' uses: docker/setup-buildx-action@v3 - name: Login to Gitea @@ -436,6 +528,8 @@ jobs: file: ./Dockerfile.shared-httpd platforms: linux/amd64 push: true + build-args: | + PKG_CACHE=${{ steps.pkgcache.outputs.url }} tags: | repo.anhonesthost.net/cloud-hosting-platform/shared-httpd:latest @@ -445,10 +539,40 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if + # it answers over TLS with our pinned CA; otherwise (down, unreachable, + # bad/expired cert) this job builds from the upstream repos and Docker Hub + # exactly as before. The repo cache and the registry mirror are probed + # separately so either can fail back on its own. + - name: Probe package cache + id: pkgcache + run: | + url=""; mirror="" + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi + if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi + echo "pkg-cache: repos=${url:-} registry-mirror=${mirror:-}" + [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "mirror=${mirror}" >> "$GITHUB_OUTPUT" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + # BuildKit (docker-container driver) ignores dockerd's registry-mirrors, + # so the mirror is configured on the builder; BuildKit itself still falls + # back to Docker Hub if the mirror stops answering mid-run. + - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror) + if: steps.pkgcache.outputs.mirror != '' + uses: docker/setup-buildx-action@v3 + with: + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["192.168.1.40:5000"] + [registry."192.168.1.40:5000"] + ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"] + - name: Set up Docker Buildx + if: steps.pkgcache.outputs.mirror == '' uses: docker/setup-buildx-action@v3 - name: Login to Gitea @@ -469,5 +593,6 @@ jobs: build-args: | PHPVER=83 OLS_VERSION=1.8.4 + PKG_CACHE=${{ steps.pkgcache.outputs.url }} tags: | repo.anhonesthost.net/cloud-hosting-platform/shared-ols:latest diff --git a/docs/pkg-cache.md b/docs/pkg-cache.md index f476dcd..ceae8fa 100644 --- a/docs/pkg-cache.md +++ b/docs/pkg-cache.md @@ -95,72 +95,90 @@ Remaining risks: - It passes the results as `EXPECT_OPENSSL_MIN` / `EXPECT_HTTPD_MIN` / `EXPECT_IMAGEMAGICK_MIN`. - The Dockerfile asserts `rpm -q --qf '%{EVR}'` is not older (`rpm --eval "%{lua: print(rpm.vercmp('A','B'))}"`). -## CI integration plan (NOT applied — needs Josh) +## CI integration (APPLIED 2026-10-21) -1. **Network (blocker).** The ephemeral runners are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40: +1. **Runner network.** Ephemeral runners are NATed and `iptables FORWARD` drops + all RFC1918 destinations: - `eph-pve-*` on `pve`, vmbr2, 10.99.98.0/24; - `eph-pve3-*` on `pve3`, vmbr1, 10.99.99.0/24. - A narrow exception is needed, placed before the DROP lines: + One rule per node now sits **above** those DROPs. It is persistent through a + `post-up`/`post-down` pair placed before the first DROP line in + `/etc/network/interfaces.d/vmbr2-ci-isolated` (pve) and + `/etc/network/interfaces.d/vmbr1-ci-isolated` (pve3). The Proxmox firewall is + disabled on both nodes, so its reloads do not touch these chains. Backups: + `/root/vmbr{2,1}-ci-isolated.bak-pkgcache-20261021`. ``` - # pve, /etc/network/interfaces.d/ vmbr2 stanza, BEFORE the 192.168.0.0/16 DROP: - post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT - post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT - # live: iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT - # pve3: the same with vmbr1 + iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40/32 -p tcp -m multiport --dports 443,5000 \ + -m comment --comment pkg-cache-ct130 -j ACCEPT ``` - `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach the cache is untested. -2. **Probe with the CA, then opt in.** Keep `${{ }}` out of `run:`. The probe fails when the cache is down **or its certificate is expired or untrusted**, and the build then uses upstream. - ```yaml - - name: Probe package cache - id: pkgcache - run: | - if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/ ; then - echo "url=https://192.168.1.40" >> "$GITHUB_OUTPUT" - else - echo "url=" >> "$GITHUB_OUTPUT"; echo "::warning::pkg-cache unreachable - building from upstream" - fi - # docker/build-push-action `build-args:` add PKG_CACHE=${{ steps.pkgcache.outputs.url }} - ``` -3. **Docker Hub mirror for BuildKit**, over TLS and without `insecure`. `setup-buildx-action` uses the docker-container driver, which **ignores dockerd's `registry-mirrors`**. buildx copies the `ca` file into the builder. - ```yaml - - uses: docker/setup-buildx-action@v3 - with: - buildkitd-config-inline: | - [registry."docker.io"] - mirrors = ["192.168.1.40:5000"] - [registry."192.168.1.40:5000"] - ca = ["keys/pkg-cache/ca.crt"] - ``` - The exact equivalent (absolute `ca` path) was measured on 192.168.1.193. BuildKit falls back to Docker Hub when the mirror is down; this was measured too, with the registry stopped. The relative `ca` path inside a workflow is **untested**; use `${{ github.workspace }}/keys/pkg-cache/ca.crt` if it doesn't resolve. -4. **dockerd clients** (plain `docker pull`). Untested, because it needs a daemon restart. Use TLS, no `insecure-registries`: + Verified from throwaway CTs on vmbr2 and vmbr1: + - :443 and :5000 work with TLS verified; + - CT130 :22/:80, pve :8006, .193 :22, the router's :53 and 10.10.14.20 stay **blocked**; + - internet access is unchanged. + + The LAN runner VM *Git-Builders* (VM103, 192.168.1.126 on vmbr0) needs no + rule. `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which + is unroutable from both the LAN and the DC (stale record?). +2. **Workflow** (`.gitea/workflows/build-push.yaml`). Jobs Build-and-Push, + Build-FPM-Images, Build-Shared-httpd and Build-Shared-OLS run a + **"Probe package cache"** step. It curls `https://192.168.1.40/` and + `https://192.168.1.40:5000/v2/` with `--cacert keys/pkg-cache/ca.crt + --max-time 5`. + - If both answer: `PKG_CACHE=https://192.168.1.40` is passed as a build arg, + and buildx is set up with the Docker Hub mirror (`buildkitd-config-inline`, + `ca=`). + - If either fails: that output is empty, the build uses upstream repos / + Docker Hub, and a `::warning::` is emitted. + + Litespeed and lsphp jobs are unchanged (their Dockerfiles are not wired). + **Fail-back measured** with the workflow's own step code, from a runner-bridge CT: + + | cache state | probe result | time | + |---|---|---| + | up | url + mirror set | <1 s | + | nginx + registry stopped | both empty, warning | <1 s (connection refused) | + | silently dropped | both empty, warning | 10 s (2 x 5 s) | + + Mid-build cache loss is also covered: dnf has the origin as a 2nd baseurl, and + BuildKit falls back from the mirror to Docker Hub (measured with the registry stopped). +3. **dockerd clients** (plain `docker pull`). Untested, because it needs a daemon restart. + Use TLS, no `insecure-registries`: ``` install -D -m0644 keys/pkg-cache/ca.crt /etc/docker/certs.d/192.168.1.40:5000/ca.crt # /etc/docker/daemon.json: { "registry-mirrors": ["https://192.168.1.40:5000"] } then restart docker ``` -## WHP hosts (plan only) +## WHP hosts (`scripts/host/pkgcache-host-repos.sh`) -The production hosts (whp01, whp02, sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts can, e.g. 192.168.1.148. Options for Josh: -- (a) a second identical cache in the datacenter, with its own name-constrained CA; -- (b) expose this one over NetBird; -- (c) LAN hosts only. +`pkgcache-host-repos.sh apply https://192.168.1.40` does the following: +- backs up every `/etc/yum.repos.d/*.repo` to `/root/pkgcache-repo-backup-/`; +- installs the name-constrained CA as a system anchor; +- rewrites the AlmaLinux, EPEL and Remi sections to `baseurl=/... /...`; +- comments out mirrorlist/metalink in those sections; +- sets `gpgcheck=1`, plus `repo_gpgcheck=1` for AlmaLinux and Remi. -For a host that can reach a cache: -- install the CA at `/etc/pki/pkg-cache/ca.crt`; -- create a bundle (`cat /etc/pki/tls/certs/ca-bundle.crt /etc/pki/pkg-cache/ca.crt > /etc/pki/pkg-cache/bundle.pem`); -- in each repo section, keep the origin as the second URL: +`revert [DIR]` restores byte-identical files (verified on AL9 and AL10) and +removes the anchor. `status` shows the current state. -```ini -[remi-safe] -baseurl=https:///remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/ -#mirrorlist=... -sslcacert=/etc/pki/pkg-cache/bundle.pem -gpgcheck=1 -repo_gpgcheck=1 # Remi + AlmaLinux sign repomd; EPEL does not -``` +**Why not "cache first, mirrorlist as fallback":** measured on AL10, with +mirrorlist/metalink left in place dnf **ignores `baseurl` completely** (0 cache +requests). The origin as second baseurl is the fallback dnf really performs. -Hosts have no php.net gate. On a security-release day, run `pkgcache-refresh` on the cache and check `dnf info php-cli` against php.net. +**Measured on 192.168.1.148** (AlmaLinux 10.2), 2026-10-21: + +| test | result | +|---|---| +| baseline `dnf makecache --refresh` (original mirrorlists) | rc=0, **1209 s** | +| via cache `makecache --refresh` | rc=0, **22 s** (2nd run, non-interactive: 2 s) | +| via cache `dnf download php-cli` | rc=0, 1 s | +| `dnf -y --downloadonly update php-cli` | "Nothing to do" (already 8.5.12) | +| cache **down** (nginx stopped): makecache --refresh / download | rc=0 **16 s** / rc=0 24 s, served by origin | +| cache **silently dropped** (iptables DROP): makecache --refresh / download | rc=0 **225 s** / rc=0 59 s (dnf's 30 s connect timeout per attempt) | + +**Datacenter hosts** (whp01, whp02, sdbees, TrueSelfCA) cannot reach +192.168.1.40 and are **not configured**. That needs CT130 on NetBird (see the +report: setup key / route pending). ## Operating the cache (CT130) diff --git a/scripts/host/pkgcache-host-repos.sh b/scripts/host/pkgcache-host-repos.sh new file mode 100644 index 0000000..2f44b04 --- /dev/null +++ b/scripts/host/pkgcache-host-repos.sh @@ -0,0 +1,107 @@ +#!/bin/bash +## pkgcache-host-repos.sh — point a WHP HOST's dnf repos (AlmaLinux 9/10 BaseOS/ +## AppStream/CRB/extras/..., EPEL, Remi) at the internal package cache, with the +## ORIGIN as automatic fallback. Not used by image builds (see pkg-cache-repos.sh). +## +## pkgcache-host-repos.sh apply https://192.168.1.40 # backs up, then rewrites +## pkgcache-host-repos.sh status +## pkgcache-host-repos.sh revert [BACKUP_DIR] # default: newest backup +## +## Fallback design (measured 2026-10-21 on AlmaLinux 10.2): +## * each rewritten section gets baseurl=// / +## — dnf/librepo treats the second URL as a failover mirror. +## * the section's mirrorlist=/metalink= is COMMENTED OUT. Measured: with a +## mirrorlist/metalink present dnf ignores baseurl entirely (0 cache requests), +## so "cache first, mirrorlist as fallback" is not something dnf can do. +## * gpgcheck=1 everywhere; repo_gpgcheck=1 for AlmaLinux + Remi (both sign +## repomd.xml); EPEL has no repomd signature and relies on HTTPS end to end. +## * trust: the name-constrained pkg-cache CA is installed as a system anchor +## (/etc/pki/ca-trust/source/anchors/pkg-cache-ca.crt). Its nameConstraints +## (IP 192.168.1.40, pkg-cache.* DNS) are enforced by OpenSSL — a cert it +## signs for any other name fails ("permitted subtree violation"). +## Every changed file is copied to /root/pkgcache-repo-backup-/ first. +set -euo pipefail + +MODE="${1:-status}" +CA_SRC="$(dirname "$0")/pkg-cache-ca.crt" +ANCHOR=/etc/pki/ca-trust/source/anchors/pkg-cache-ca.crt +die() { echo "FATAL(pkgcache-host-repos): $*" >&2; exit 1; } +same() { [ "$(sha256sum < "$1")" = "$(sha256sum < "$2")" ]; } # cmp/diffutils is not on minimal hosts + +rewrite() { # $1 file $2 cache url + python3 - "$1" "$2" <<'PY' +import re, sys +f, C = sys.argv[1], sys.argv[2].rstrip('/') +maps = [ # regex on a (commented or live) baseurl, cache prefix, origin base, repomd signed? + (r'^#?\s*baseurl\s*=\s*https?://repo\.almalinux\.org/almalinux/(\S+)', 'almalinux', 'https://repo.almalinux.org/almalinux/', True), + (r'^#?\s*baseurl\s*=\s*https?://download\.example/pub/epel/(\S+)', 'epel', 'https://dl.fedoraproject.org/pub/epel/', False), + (r'^#?\s*baseurl\s*=\s*https?://rpms\.remirepo\.net/(\S+)', 'remi', 'https://rpms.remirepo.net/', True), +] +out, sec = [], [] +state = {'hit': False, 'sig': False} +def flush(): + for x in sec: + if state['hit']: + if re.match(r'^(mirrorlist|metalink)\s*=', x): + x = '#' + x + elif re.match(r'^(gpgcheck|repo_gpgcheck)\s*=', x): + continue + out.append(x) + if state['hit']: + # keep blank-line layout: insert before trailing blanks + tail = [] + while out and out[-1].strip() == '': + tail.append(out.pop()) + out.append('gpgcheck=1') + if state['sig']: + out.append('repo_gpgcheck=1') + out.extend(tail) + sec.clear(); state['hit'] = state['sig'] = False +for line in open(f).read().split('\n'): + if line.startswith('['): + flush() + for rx, prefix, origin, sig in maps: + m = re.match(rx, line) + if m: + p = m.group(1) + line = f'baseurl={C}/{prefix}/{p} {origin}{p}' + state['hit'] = True; state['sig'] = state['sig'] or sig + break + sec.append(line) +flush() +open(f, 'w').write('\n'.join(out)) +PY +} + +case "$MODE" in + apply) + C="${2:?usage: apply https://}"; C="${C%/}" + case "$C" in https://*) ;; *) die "cache URL must be https://" ;; esac + [ -s "$CA_SRC" ] || die "missing $CA_SRC (copy keys/pkg-cache/ca.crt next to this script)" + B=/root/pkgcache-repo-backup-$(date -u +%Y%m%dT%H%M%SZ) + mkdir -p "$B/yum.repos.d" + cp -a /etc/yum.repos.d/*.repo "$B/yum.repos.d/" + [ -e "$ANCHOR" ] && cp -a "$ANCHOR" "$B/" || touch "$B/.no-anchor-before" + install -m 0644 "$CA_SRC" "$ANCHOR" && update-ca-trust + for f in /etc/yum.repos.d/almalinux*.repo /etc/yum.repos.d/epel*.repo /etc/yum.repos.d/remi*.repo; do + [ -f "$f" ] || continue + grep -q -E '^#?\s*baseurl\s*=\s*https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" && rewrite "$f" "$C" + done + echo "backup: $B" + echo "changed files:"; for f in "$B"/yum.repos.d/*.repo; do same "$f" "/etc/yum.repos.d/$(basename "$f")" || echo " /etc/yum.repos.d/$(basename "$f")"; done + ;; + revert) + B="${2:-$(ls -d /root/pkgcache-repo-backup-* 2>/dev/null | sort | tail -1)}" + [ -d "$B/yum.repos.d" ] || die "no backup dir ($B)" + for f in "$B"/yum.repos.d/*.repo; do same "$f" "/etc/yum.repos.d/$(basename "$f")" || { cp -a "$f" /etc/yum.repos.d/; echo "restored /etc/yum.repos.d/$(basename "$f")"; }; done + if [ -e "$B/.no-anchor-before" ]; then rm -f "$ANCHOR"; update-ca-trust; echo "removed $ANCHOR"; fi + dnf clean metadata -q || true + echo "reverted from $B" + ;; + status) + grep -l -E '^baseurl=https://[^ ]+/(almalinux|epel|remi)/' /etc/yum.repos.d/*.repo 2>/dev/null | sed 's/^/via cache: /' || echo "no repo uses the cache" + ls -d /root/pkgcache-repo-backup-* 2>/dev/null | sed 's/^/backup: /' || true + [ -e "$ANCHOR" ] && echo "CA anchor: $ANCHOR" || echo "CA anchor: absent" + ;; + *) die "usage: apply URL | status | revert [DIR]" ;; +esac -- 2.52.0