Merge pull request 'feat(build): opt-in PKG_CACHE build arg for the internal package cache (HTTPS, GPG-verified)' (#31) from feat/pkg-cache-build-arg into trunk

.gitea/workflows/build-push.yaml

@@ -85,10 +85,40 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if
+      # it answers over TLS with our pinned CA; otherwise (down, unreachable,
+      # bad/expired cert) this job builds from the upstream repos and Docker Hub
+      # exactly as before. The repo cache and the registry mirror are probed
+      # separately so either can fail back on its own.
+      - name: Probe package cache
+        id: pkgcache
+        run: |
+          url=""; mirror=""
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi
+          echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=${mirror:-<unreachable: Docker Hub>}"
+          [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
+          echo "url=${url}" >> "$GITHUB_OUTPUT"
+          echo "mirror=${mirror}" >> "$GITHUB_OUTPUT"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
+      # BuildKit (docker-container driver) ignores dockerd's registry-mirrors,
+      # so the mirror is configured on the builder; BuildKit itself still falls
+      # back to Docker Hub if the mirror stops answering mid-run.
+      - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror)
+        if: steps.pkgcache.outputs.mirror != ''
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["192.168.1.40:5000"]
+            [registry."192.168.1.40:5000"]
+              ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"]
+
       - name: Set up Docker Buildx
+        if: steps.pkgcache.outputs.mirror == ''
         uses: docker/setup-buildx-action@v3
 
       # Per-run cache-bust value. Each job's buildx builder is created fresh by
@@ -135,6 +165,7 @@ jobs:
           build-args: |
             PHPVER=${{ matrix.phpver }}
             CACHE_BUST=${{ steps.bust.outputs.value }}
+            PKG_CACHE=${{ steps.pkgcache.outputs.url }}
             EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }}
           tags: |
             repo.anhonesthost.net/cloud-hosting-platform/cac:php${{ matrix.phpver }}
@@ -152,10 +183,40 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if
+      # it answers over TLS with our pinned CA; otherwise (down, unreachable,
+      # bad/expired cert) this job builds from the upstream repos and Docker Hub
+      # exactly as before. The repo cache and the registry mirror are probed
+      # separately so either can fail back on its own.
+      - name: Probe package cache
+        id: pkgcache
+        run: |
+          url=""; mirror=""
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi
+          echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=${mirror:-<unreachable: Docker Hub>}"
+          [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
+          echo "url=${url}" >> "$GITHUB_OUTPUT"
+          echo "mirror=${mirror}" >> "$GITHUB_OUTPUT"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
+      # BuildKit (docker-container driver) ignores dockerd's registry-mirrors,
+      # so the mirror is configured on the builder; BuildKit itself still falls
+      # back to Docker Hub if the mirror stops answering mid-run.
+      - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror)
+        if: steps.pkgcache.outputs.mirror != ''
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["192.168.1.40:5000"]
+            [registry."192.168.1.40:5000"]
+              ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"]
+
       - name: Set up Docker Buildx
+        if: steps.pkgcache.outputs.mirror == ''
         uses: docker/setup-buildx-action@v3
 
       # Per-run cache-bust value. Each job's buildx builder is created fresh by
@@ -203,6 +264,7 @@ jobs:
           build-args: |
             PHPVER=${{ matrix.phpver }}
             CACHE_BUST=${{ steps.bust.outputs.value }}
+            PKG_CACHE=${{ steps.pkgcache.outputs.url }}
             EXPECT_PHP_MIN=${{ steps.phpmin.outputs.min }}
           tags: |
             repo.anhonesthost.net/cloud-hosting-platform/cac-fpm:php${{ matrix.phpver }}
@@ -417,10 +479,40 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if
+      # it answers over TLS with our pinned CA; otherwise (down, unreachable,
+      # bad/expired cert) this job builds from the upstream repos and Docker Hub
+      # exactly as before. The repo cache and the registry mirror are probed
+      # separately so either can fail back on its own.
+      - name: Probe package cache
+        id: pkgcache
+        run: |
+          url=""; mirror=""
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi
+          echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=${mirror:-<unreachable: Docker Hub>}"
+          [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
+          echo "url=${url}" >> "$GITHUB_OUTPUT"
+          echo "mirror=${mirror}" >> "$GITHUB_OUTPUT"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
+      # BuildKit (docker-container driver) ignores dockerd's registry-mirrors,
+      # so the mirror is configured on the builder; BuildKit itself still falls
+      # back to Docker Hub if the mirror stops answering mid-run.
+      - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror)
+        if: steps.pkgcache.outputs.mirror != ''
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["192.168.1.40:5000"]
+            [registry."192.168.1.40:5000"]
+              ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"]
+
       - name: Set up Docker Buildx
+        if: steps.pkgcache.outputs.mirror == ''
         uses: docker/setup-buildx-action@v3
 
       - name: Login to Gitea
@@ -436,6 +528,8 @@ jobs:
           file: ./Dockerfile.shared-httpd
           platforms: linux/amd64
           push: true
+          build-args: |
+            PKG_CACHE=${{ steps.pkgcache.outputs.url }}
           tags: |
             repo.anhonesthost.net/cloud-hosting-platform/shared-httpd:latest
 
@@ -445,10 +539,40 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Internal package cache (CT130 pkg-cache, docs/pkg-cache.md). Used ONLY if
+      # it answers over TLS with our pinned CA; otherwise (down, unreachable,
+      # bad/expired cert) this job builds from the upstream repos and Docker Hub
+      # exactly as before. The repo cache and the registry mirror are probed
+      # separately so either can fail back on its own.
+      - name: Probe package cache
+        id: pkgcache
+        run: |
+          url=""; mirror=""
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
+          if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40:5000/v2/; then mirror="192.168.1.40:5000"; fi
+          echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=${mirror:-<unreachable: Docker Hub>}"
+          [ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
+          echo "url=${url}" >> "$GITHUB_OUTPUT"
+          echo "mirror=${mirror}" >> "$GITHUB_OUTPUT"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
+      # BuildKit (docker-container driver) ignores dockerd's registry-mirrors,
+      # so the mirror is configured on the builder; BuildKit itself still falls
+      # back to Docker Hub if the mirror stops answering mid-run.
+      - name: Set up Docker Buildx (Docker Hub via pkg-cache mirror)
+        if: steps.pkgcache.outputs.mirror != ''
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config-inline: |
+            [registry."docker.io"]
+              mirrors = ["192.168.1.40:5000"]
+            [registry."192.168.1.40:5000"]
+              ca = ["${{ github.workspace }}/keys/pkg-cache/ca.crt"]
+
       - name: Set up Docker Buildx
+        if: steps.pkgcache.outputs.mirror == ''
         uses: docker/setup-buildx-action@v3
 
       - name: Login to Gitea
@@ -469,5 +593,6 @@ jobs:
           build-args: |
             PHPVER=83
             OLS_VERSION=1.8.4
+            PKG_CACHE=${{ steps.pkgcache.outputs.url }}
           tags: |
             repo.anhonesthost.net/cloud-hosting-platform/shared-ols:latest

Dockerfile

@@ -7,10 +7,17 @@ ARG CACHE_BUST=
 # php.net's current release for this branch (CI); the build fails on older PHP.
 ARG EXPECT_PHP_MIN=
 
+# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
+# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
+# GPG-verified against keys/pkg-cache/ either way. Repo files
+# are restored at the end of every RUN, so no layer keeps the cache address.
+# See scripts/pkg-cache-repos.sh.
+ARG PKG_CACHE=
+COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
+COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
 # Install repos, update, install only needed packages, clean up in one layer
-RUN dnf install -y \
-      https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm \
-      https://rpms.remirepo.net/enterprise/remi-release-10.rpm && \
+RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
+    /usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \
     # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security
     # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out),
     # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback
@@ -18,10 +25,12 @@ RUN dnf install -y \
     # replayed (old) http index can't ship old PHP: packages stay GPG-checked and
     # the EXPECT_PHP_MIN gate below fails the build on anything older than php.net.
     sed -i -e 's|^#baseurl=http://\(.*\)$|baseurl=https://\1 http://\1|' -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/remi*.repo && \
+    /usr/local/sbin/pkg-cache-repos.sh rpm && \
     printf 'retries=10\ntimeout=60\n' >> /etc/dnf/dnf.conf && \
     dnf update -y && \
     dnf install -y httpd mod_ssl openssl wget procps cronie iproute postgresql-devel microdnf less git \
       nano rsync unzip zip mariadb bind-utils jq patch nc tree dos2unix && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     dnf clean all && \
     rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/*
 
@@ -30,13 +39,17 @@ COPY ./scripts/ /scripts/
 RUN chmod +x /scripts/*
 
 # Install ImageMagick from EPEL with HEIC/HEIF/AVIF support
-RUN dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \
+RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
+    dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     dnf clean all
 
 # Generate self-signed cert, create needed dirs, install PHP, clean up
 RUN openssl req -newkey rsa:2048 -nodes -keyout /etc/pki/tls/private/localhost.key -x509 -days 3650 -subj "/CN=localhost" -out /etc/pki/tls/certs/localhost.crt && \
     mkdir -p /run/php-fpm/ && \
+    /usr/local/sbin/pkg-cache-repos.sh rpm && \
     /scripts/install-php$PHPVER.sh && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     # CI passes php.net's current release for the branch; never ship older PHP.
     if [ -n "$EXPECT_PHP_MIN" ]; then \
       php -r 'exit(version_compare(PHP_VERSION, getenv("EXPECT_PHP_MIN"), ">=") ? 0 : 1);' || \

Dockerfile.fpm

@@ -7,10 +7,17 @@ ARG CACHE_BUST=
 # php.net's current release for this branch (CI); the build fails on older PHP.
 ARG EXPECT_PHP_MIN=
 
+# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
+# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
+# GPG-verified against keys/pkg-cache/ either way. Repo files
+# are restored at the end of every RUN, so no layer keeps the cache address.
+# See scripts/pkg-cache-repos.sh.
+ARG PKG_CACHE=
+COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
+COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
 # Install repos, update, install only needed packages (no httpd/mod_ssl), clean up in one layer
-RUN dnf install -y \
-      https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm \
-      https://rpms.remirepo.net/enterprise/remi-release-10.rpm && \
+RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
+    /usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \
     # Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security
     # release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out),
     # so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback
@@ -18,10 +25,12 @@ RUN dnf install -y \
     # replayed (old) http index can't ship old PHP: packages stay GPG-checked and
     # the EXPECT_PHP_MIN gate below fails the build on anything older than php.net.
     sed -i -e 's|^#baseurl=http://\(.*\)$|baseurl=https://\1 http://\1|' -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/remi*.repo && \
+    /usr/local/sbin/pkg-cache-repos.sh rpm && \
     printf 'retries=10\ntimeout=60\n' >> /etc/dnf/dnf.conf && \
     dnf update -y && \
     dnf install -y openssl wget procps cronie iproute postgresql-devel microdnf less git \
       nano rsync unzip zip mariadb bind-utils jq patch nc tree dos2unix fcgi && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     dnf clean all && \
     rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/*
 
@@ -30,12 +39,16 @@ COPY ./scripts/ /scripts/
 RUN chmod +x /scripts/*
 
 # Install ImageMagick from EPEL with HEIC/HEIF/AVIF support
-RUN dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \
+RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
+    dnf install -y ImageMagick ImageMagick-libs ImageMagick-heic && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     dnf clean all
 
 # Create needed dirs, install PHP, clean up (no SSL cert, no httpd)
 RUN mkdir -p /run/php-fpm/ && \
+    /usr/local/sbin/pkg-cache-repos.sh rpm && \
     /scripts/install-php$PHPVER.sh && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     # CI passes php.net's current release for the branch; never ship older PHP.
     if [ -n "$EXPECT_PHP_MIN" ]; then \
       php -r 'exit(version_compare(PHP_VERSION, getenv("EXPECT_PHP_MIN"), ">=") ? 0 : 1);' || \

Dockerfile.shared-httpd

@@ -1,10 +1,21 @@
 FROM almalinux/10-base
 
+# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
+# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
+# GPG-verified against keys/pkg-cache/ either way. Repo files
+# are restored at the end of every RUN, so no layer keeps the cache address.
+# See scripts/pkg-cache-repos.sh.
+ARG PKG_CACHE=
+COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
+COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
+
 # Install Apache and minimal dependencies (no PHP at all)
-RUN dnf install -y \
-      https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm && \
+RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
+    /usr/local/sbin/pkg-cache-repos.sh release-install epel && \
+    /usr/local/sbin/pkg-cache-repos.sh rpm && \
     dnf update -y && \
     dnf install -y httpd mod_ssl openssl iproute cronie procps curl && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     dnf clean all && \
     rm -rf /var/cache/dnf /usr/share/doc /usr/share/man /usr/share/locale/*
 

Dockerfile.shared-ols

@@ -19,6 +19,15 @@ ARG OLS_VERSION=1.8.4
 ARG PHPVER=83
 FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER}
 
+## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
+## (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
+## GPG-verified against keys/pkg-cache/ either way. Repo files
+## are restored at the end of every RUN, so no layer keeps the cache address.
+## See scripts/pkg-cache-repos.sh.
+ARG PKG_CACHE=
+COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
+COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
+
 ## Tooling the shared tier needs on top of the base:
 ##   - inotify-tools: the .htaccess watcher (spec 5.3)
 ##   - gettext-base: envsubst for render-shared-ols-config.sh
@@ -28,9 +37,11 @@ FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER}
 ##     liveness check depends on. Only transitively present via the base image
 ##     today (Ubuntu 24.04 pulls it in) — pin it explicitly so it can't be
 ##     pruned as "unused" and silently break the supervisor's crash detection.
-RUN apt-get update && \
+RUN /usr/local/sbin/pkg-cache-repos.sh apt && \
+    apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         inotify-tools gettext-base openssl ca-certificates curl procps && \
+    /usr/local/sbin/pkg-cache-repos.sh restore && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 

docs/pkg-cache.md

@@ -0,0 +1,208 @@
+# Internal package + image cache (`PKG_CACHE`)
+
+Opt-in build support for the LAN package cache **CT130 `pkg-cache` at `https://192.168.1.40`** (Proxmox node `pve`, storage `shared-storage-tn`).
+
+Without `PKG_CACHE`, builds use the upstream repositories. The one change from before is that the EPEL and Remi release RPMs are now **GPG-verified** against `keys/pkg-cache/` before they are installed.
+
+Why it exists: during the 2026-10-21 PHP security release, builds failed because:
+- Remi mirrors lagged hours behind origin;
+- Remi's origin reset or timed out HTTPS under load;
+- an EPEL mirror had a broken certificate.
+
+On that day a build without the cache spent **1031 s** in its first dnf step. Through the cache the same step took **40 s cold and 21 s warm**.
+
+## What the cache serves
+
+| URL (https, pkg-cache CA) | Upstream tiers, tried in order (stale copy last) |
+|---|---|
+| `https://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) |
+| `https://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` |
+| `https://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` |
+| `https://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` |
+| `https://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` |
+| `https://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` |
+| `192.168.1.40:5000` (TLS, pkg-cache CA) | Docker Hub pull-through cache (`registry-1.docker.io`) |
+
+Plain `http://192.168.1.40/` answers **403** to LAN clients.
+
+Cache rules:
+- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata means:
+  - `repomd.xml*`
+  - repodata files whose names don't start with a hash
+  - apt `dists/`, except `by-hash/`
+  - `*-latest*.rpm` and `remi-release-N.rpm`
+  - `RPM-GPG-KEY*`
+  - directory listings
+- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are cached for 30 days and evicted after 60 days unused. Total size is capped at 110 GB, with 15 GB kept free.
+- On a connect error, timeout or 5xx, the request falls through to the next tier. Only the **last** tier may answer from a stale copy, so the order is primary, then fallback, then stale.
+- `proxy_cache_lock` is on.
+- Responses carry `X-Cache-Status` and `X-Pkgcache-Tier`.
+
+## Building with the cache
+
+```bash
+docker buildx build --build-arg PKG_CACHE=https://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile .
+```
+
+`scripts/pkg-cache-repos.sh` (installed as `/usr/local/sbin/`) and `keys/pkg-cache/` (installed as `/usr/local/share/pkg-cache/`) are wired into `Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`. The script has four modes.
+
+**`rpm`** handles AlmaLinux, EPEL and Remi:
+- Each `baseurl` becomes `<cache>/<prefix>/... <origin>/...`. dnf fails over to the origin if the cache dies.
+- That section's mirrorlist/metalink is commented out.
+- The section gets `sslcacert=` pointing at a bundle of the system CAs plus the pkg-cache CA. The bundle lives in the state dir and is deleted by `restore`.
+- It also gets `gpgcheck=1`.
+- AlmaLinux and Remi also get `repo_gpgcheck=1`, because both sign `repomd.xml` (measured 2026-10-21: `repomd.xml.asc` returns 200). EPEL publishes no repomd signature (404), so its metadata integrity rests on HTTPS end to end.
+
+**`apt`** handles Ubuntu archive/security and LiteSpeed:
+- Their URIs are rewritten to the cache.
+- `/etc/apt/apt.conf.d/99pkg-cache` sets `Acquire::https::192.168.1.40::CaInfo`, so the CA is trusted for the cache host only.
+- apt verifies the signed `InRelease`/`Release.gpg` as always. It has no failover, so CI must probe the cache first.
+
+**`release-install epel [remi]`**:
+- Downloads the release RPM over HTTPS, from the cache or from the origin.
+- Checks the package name.
+- Imports the key committed in `keys/pkg-cache/`.
+- Requires `rpm -K` to report `digests signatures OK`.
+- Installs with `--setopt=localpkg_gpgcheck=1`.
+
+**`restore`**:
+- Puts rewritten files back.
+- Deletes files the script created.
+- Deletes any repo/apt file that **appeared during the RUN** and still references the cache.
+- Then **fails the build** if any file under `/etc/yum.repos.d`, `/etc/dnf` or `/etc/apt` still mentions the cache.
+
+Every RUN that rewrites ends with `restore`, so no shipped layer carries the LAN address. This was verified on the built images: 0 references, no state dir, no apt drop-in.
+
+`PKG_CACHE` must start with `https://`. `http://` is refused.
+
+`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Two things are needed first:
+- the multi-stage `lsphp-apt-upgrade.sh` flow needs a `PKG_CACHE` ARG in each stage;
+- its madison check must accept the cache URL. It currently requires `*rpms.litespeedtech.com*`, and via the cache madison shows `https://192.168.1.40/litespeed/debian`.
+
+## Trust model
+
+`keys/pkg-cache/ca.crt` is the *pkg-cache internal CA*:
+- EC P-256, valid to 2036.
+- **Name-constrained** to IP `192.168.1.40` and DNS `pkg-cache.internal.cloud-hosting.io` / `pkg-cache.geekhome.local`. Even its private key, which exists only on CT130 at `/etc/pkgcache-ca/ca.key`, cannot mint a certificate that clients would accept for any other host.
+- It is trusted only by the rewritten repos, the apt per-host option and the BuildKit/dockerd registry config. It is never added to a system trust store.
+
+The server certificate `/etc/pkgcache-ca/server.crt` **expires 2029-10-20**. `pkgcache-stats` prints the expiry. Renew it with the same `openssl x509 -req ... -extfile srv.cnf` steps in `/etc/pkgcache-ca/`.
+
+Remaining risks:
+- **Replay of old but validly signed metadata.** Signatures stop tampering, not freshness. The cache's own plain-http tiers for Remi and LiteSpeed are the leg where an internet MITM could replay. The php.net gates (`EXPECT_PHP_MIN`, `EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`) fail an image build that gets old PHP. **Josh to decide** whether to drop those http tiers: you gain replay resistance and lose availability when Remi's HTTPS is down, as it was all day on 2026-10-21.
+- **Proposed, not implemented:** the same kind of minimum-version gate for other security-relevant packages:
+  - CI resolves the newest `openssl`/`openssl-libs` and `httpd` from AlmaLinux BaseOS/AppStream, and `ImageMagick` from EPEL, **directly from the origin over HTTPS** (e.g. `dnf repoquery --repofrompath=o,https://repo.almalinux.org/almalinux/10/BaseOS/x86_64/os/ --repo=o --latest-limit=1 --qf '%{evr}' openssl-libs`).
+  - It passes the results as `EXPECT_OPENSSL_MIN` / `EXPECT_HTTPD_MIN` / `EXPECT_IMAGEMAGICK_MIN`.
+  - The Dockerfile asserts `rpm -q --qf '%{EVR}'` is not older (`rpm --eval "%{lua: print(rpm.vercmp('A','B'))}"`).
+
+## CI integration (APPLIED 2026-10-21)
+
+1. **Runner network.** Ephemeral runners are NATed and `iptables FORWARD` drops
+   all RFC1918 destinations:
+   - `eph-pve-*` on `pve`, vmbr2, 10.99.98.0/24;
+   - `eph-pve3-*` on `pve3`, vmbr1, 10.99.99.0/24.
+
+   One rule per node now sits **above** those DROPs. It is persistent through a
+   `post-up`/`post-down` pair placed before the first DROP line in
+   `/etc/network/interfaces.d/vmbr2-ci-isolated` (pve) and
+   `/etc/network/interfaces.d/vmbr1-ci-isolated` (pve3). The Proxmox firewall is
+   disabled on both nodes, so its reloads do not touch these chains. Backups:
+   `/root/vmbr{2,1}-ci-isolated.bak-pkgcache-20261021`.
+   ```
+   iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40/32 -p tcp -m multiport --dports 443,5000 \
+     -m comment --comment pkg-cache-ct130 -j ACCEPT
+   ```
+   Verified from throwaway CTs on vmbr2 and vmbr1:
+   - :443 and :5000 work with TLS verified;
+   - CT130 :22/:80, pve :8006, .193 :22, the router's :53 and 10.10.14.20 stay **blocked**;
+   - internet access is unchanged.
+
+   The LAN runner VM *Git-Builders* (VM103, 192.168.1.126 on vmbr0) needs no
+   rule. `gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which
+   is unroutable from both the LAN and the DC (stale record?).
+2. **Workflow** (`.gitea/workflows/build-push.yaml`). Jobs Build-and-Push,
+   Build-FPM-Images, Build-Shared-httpd and Build-Shared-OLS run a
+   **"Probe package cache"** step. It curls `https://192.168.1.40/` and
+   `https://192.168.1.40:5000/v2/` with `--cacert keys/pkg-cache/ca.crt
+   --max-time 5`.
+   - If both answer: `PKG_CACHE=https://192.168.1.40` is passed as a build arg,
+     and buildx is set up with the Docker Hub mirror (`buildkitd-config-inline`,
+     `ca=`).
+   - If either fails: that output is empty, the build uses upstream repos /
+     Docker Hub, and a `::warning::` is emitted.
+
+   Litespeed and lsphp jobs are unchanged (their Dockerfiles are not wired).
+   **Fail-back measured** with the workflow's own step code, from a runner-bridge CT:
+
+   | cache state | probe result | time |
+   |---|---|---|
+   | up | url + mirror set | <1 s |
+   | nginx + registry stopped | both empty, warning | <1 s (connection refused) |
+   | silently dropped | both empty, warning | 10 s (2 x 5 s) |
+
+   Mid-build cache loss is also covered: dnf has the origin as a 2nd baseurl, and
+   BuildKit falls back from the mirror to Docker Hub (measured with the registry stopped).
+3. **dockerd clients** (plain `docker pull`). Untested, because it needs a daemon restart.
+   Use TLS, no `insecure-registries`:
+   ```
+   install -D -m0644 keys/pkg-cache/ca.crt /etc/docker/certs.d/192.168.1.40:5000/ca.crt
+   # /etc/docker/daemon.json:  { "registry-mirrors": ["https://192.168.1.40:5000"] }   then restart docker
+   ```
+
+## WHP hosts (`scripts/host/pkgcache-host-repos.sh`)
+
+`pkgcache-host-repos.sh apply https://192.168.1.40` does the following:
+- backs up every `/etc/yum.repos.d/*.repo` to `/root/pkgcache-repo-backup-<UTC>/`;
+- installs the name-constrained CA as a system anchor;
+- rewrites the AlmaLinux, EPEL and Remi sections to `baseurl=<cache>/... <origin>/...`;
+- comments out mirrorlist/metalink in those sections;
+- sets `gpgcheck=1`, plus `repo_gpgcheck=1` for AlmaLinux and Remi.
+
+`revert [DIR]` restores byte-identical files (verified on AL9 and AL10) and
+removes the anchor. `status` shows the current state.
+
+**Why not "cache first, mirrorlist as fallback":** measured on AL10, with
+mirrorlist/metalink left in place dnf **ignores `baseurl` completely** (0 cache
+requests). The origin as second baseurl is the fallback dnf really performs.
+
+**Measured on 192.168.1.148** (AlmaLinux 10.2), 2026-10-21:
+
+| test | result |
+|---|---|
+| baseline `dnf makecache --refresh` (original mirrorlists) | rc=0, **1209 s** |
+| via cache `makecache --refresh` | rc=0, **22 s** (2nd run, non-interactive: 2 s) |
+| via cache `dnf download php-cli` | rc=0, 1 s |
+| `dnf -y --downloadonly update php-cli` | "Nothing to do" (already 8.5.12) |
+| cache **down** (nginx stopped): makecache --refresh / download | rc=0 **16 s** / rc=0 24 s, served by origin |
+| cache **silently dropped** (iptables DROP): makecache --refresh / download | rc=0 **225 s** / rc=0 59 s (dnf's 30 s connect timeout per attempt) |
+
+**Datacenter hosts** (whp01, whp02, sdbees, TrueSelfCA) cannot reach
+192.168.1.40 and are **not configured**. That needs CT130 on NetBird (see the
+report: setup key / route pending).
+
+## Operating the cache (CT130)
+
+- **nginx config.**
+  - `/etc/nginx/conf.d/pkgcache-http.conf` holds the cache zone, maps and log format.
+  - `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf` hold the shared settings.
+  - `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`. Edit the `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`.
+  - Listeners: :443 TLS for the LAN, `127.0.0.1:80` for local tools, and :80 for the LAN answers 403.
+- **Registry.** Config is `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`, with TLS). Service `docker-registry`.
+- **TLS.** Everything is in `/etc/pkgcache-ca/`: `ca.key` (0600), `ca.crt`, `server.key`, `server-chain.crt`, and the `ca.cnf`/`srv.cnf` openssl configs.
+- **Data.** nginx uses `/srv/cache/nginx`, the registry uses `/srv/cache/registry`. Both live on the 150 GB mount point `mp0` (`backup=0`).
+- **Security-release day.** `pkgcache-refresh` re-fetches every cached metadata entry right away. Variants:
+  - `pkgcache-refresh remi` refreshes one prefix;
+  - `--list` lists the entries;
+  - `--purge` deletes them instead.
+
+  A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Metadata is at most 3 minutes old anyway.
+- **Monitoring.** `pkgcache-stats` shows:
+  - disk usage;
+  - hit ratio;
+  - fallback-tier use;
+  - the STALE count;
+  - upstream errors;
+  - certificate expiry.
+
+  Logs are `/var/log/nginx/pkgcache.access.log` (fields `cache=`, `tier=`, `up=`) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks are `https://192.168.1.40/` and `https://192.168.1.40:5000/v2/`, with certificate checking against the CA.
+- **Docker Hub rate limits.** Anonymous pulls are limited per egress IP, and all clients share it. Add `username`/`password` (a Hub access token) under `proxy:` if needed.

keys/pkg-cache/README

@@ -0,0 +1,20 @@
+Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh,
+docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing
+here is added to the system trust store.
+
+ca.crt                 "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18.
+                       NAME-CONSTRAINED to IP 192.168.1.40 and DNS
+                       pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local.
+                       SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3
+                       (private key lives only on CT130: /etc/pkgcache-ca/ca.key)
+RPM-GPG-KEY-EPEL-10    Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158
+                       fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and
+                       mirror.math.princeton.edu (identical); fingerprint matches
+                       https://fedoraproject.org/security/
+RPM-GPG-KEY-remi2024   Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4
+                       fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical
+                       to the key shipped in remi-release-10.2-1.el10.remi. Not found on
+                       keys.openpgp.org / keyserver.ubuntu.com - confirm out of band.
+                       Remi rotates keys (remi2025, ...): when remi-release starts being
+                       signed by a new key, `release-install remi` fails loudly - add the
+                       new key here (and to pkg-cache-repos.sh).

keys/pkg-cache/RPM-GPG-KEY-EPEL-10

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA
+Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK
+Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L
+URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf
+mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2
+XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI
+uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60
+GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix
+fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z
+BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77
+HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB
+tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB
+CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC
+BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p
+l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp
+yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg
+Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u
+scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs
+7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK
+F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg
+sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB
+0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY
+1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt
+TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k
+p9Dfd2pKuxK9Mys=
+=mhQZ
+-----END PGP PUBLIC KEY BLOCK-----

keys/pkg-cache/RPM-GPG-KEY-remi2024

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W
+Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe
+UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa
+qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN
+ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB
+jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO
+/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL
+yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ
+w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN
+0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l
+ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB
+tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly
+ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd
+/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX
+gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx
+Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W
+4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/
+/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz
+/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc
+3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof
+ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS
+dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy
+sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n
+i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd
+DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P
+7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI
+T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ
+OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3
+cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W
+RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x
+5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m
+8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD
+pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r
+N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm
+Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG
+Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA
+AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9
+OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI
+wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA
+qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c
+WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV
+u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI
+qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S
+4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4
+MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II
+Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl
+bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s
+r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1
+=meaU
+-----END PGP PUBLIC KEY BLOCK-----

keys/pkg-cache/ca.crt

@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw
+TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV
+BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN
+MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg
+KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG
+ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV
+OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD
+VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/
+v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw
+a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu
+Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg
+cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM
+YAIl
+-----END CERTIFICATE-----

scripts/host/pkgcache-host-repos.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+## pkgcache-host-repos.sh — point a WHP HOST's dnf repos (AlmaLinux 9/10 BaseOS/
+## AppStream/CRB/extras/..., EPEL, Remi) at the internal package cache, with the
+## ORIGIN as automatic fallback. Not used by image builds (see pkg-cache-repos.sh).
+##
+##   pkgcache-host-repos.sh apply https://192.168.1.40   # backs up, then rewrites
+##   pkgcache-host-repos.sh status
+##   pkgcache-host-repos.sh revert [BACKUP_DIR]          # default: newest backup
+##
+## Fallback design (measured 2026-10-21 on AlmaLinux 10.2):
+##  * each rewritten section gets  baseurl=<cache>/<prefix>/<path> <origin>/<path>
+##    — dnf/librepo treats the second URL as a failover mirror.
+##  * the section's mirrorlist=/metalink= is COMMENTED OUT. Measured: with a
+##    mirrorlist/metalink present dnf ignores baseurl entirely (0 cache requests),
+##    so "cache first, mirrorlist as fallback" is not something dnf can do.
+##  * gpgcheck=1 everywhere; repo_gpgcheck=1 for AlmaLinux + Remi (both sign
+##    repomd.xml); EPEL has no repomd signature and relies on HTTPS end to end.
+##  * trust: the name-constrained pkg-cache CA is installed as a system anchor
+##    (/etc/pki/ca-trust/source/anchors/pkg-cache-ca.crt). Its nameConstraints
+##    (IP 192.168.1.40, pkg-cache.* DNS) are enforced by OpenSSL — a cert it
+##    signs for any other name fails ("permitted subtree violation").
+## Every changed file is copied to /root/pkgcache-repo-backup-<UTC>/ first.
+set -euo pipefail
+
+MODE="${1:-status}"
+CA_SRC="$(dirname "$0")/pkg-cache-ca.crt"
+ANCHOR=/etc/pki/ca-trust/source/anchors/pkg-cache-ca.crt
+die() { echo "FATAL(pkgcache-host-repos): $*" >&2; exit 1; }
+same() { [ "$(sha256sum < "$1")" = "$(sha256sum < "$2")" ]; }   # cmp/diffutils is not on minimal hosts
+
+rewrite() {  # $1 file  $2 cache url
+    python3 - "$1" "$2" <<'PY'
+import re, sys
+f, C = sys.argv[1], sys.argv[2].rstrip('/')
+maps = [  # regex on a (commented or live) baseurl, cache prefix, origin base, repomd signed?
+    (r'^#?\s*baseurl\s*=\s*https?://repo\.almalinux\.org/almalinux/(\S+)', 'almalinux', 'https://repo.almalinux.org/almalinux/', True),
+    (r'^#?\s*baseurl\s*=\s*https?://download\.example/pub/epel/(\S+)',     'epel',      'https://dl.fedoraproject.org/pub/epel/', False),
+    (r'^#?\s*baseurl\s*=\s*https?://rpms\.remirepo\.net/(\S+)',             'remi',      'https://rpms.remirepo.net/', True),
+]
+out, sec = [], []
+state = {'hit': False, 'sig': False}
+def flush():
+    for x in sec:
+        if state['hit']:
+            if re.match(r'^(mirrorlist|metalink)\s*=', x):
+                x = '#' + x
+            elif re.match(r'^(gpgcheck|repo_gpgcheck)\s*=', x):
+                continue
+        out.append(x)
+    if state['hit']:
+        # keep blank-line layout: insert before trailing blanks
+        tail = []
+        while out and out[-1].strip() == '':
+            tail.append(out.pop())
+        out.append('gpgcheck=1')
+        if state['sig']:
+            out.append('repo_gpgcheck=1')
+        out.extend(tail)
+    sec.clear(); state['hit'] = state['sig'] = False
+for line in open(f).read().split('\n'):
+    if line.startswith('['):
+        flush()
+    for rx, prefix, origin, sig in maps:
+        m = re.match(rx, line)
+        if m:
+            p = m.group(1)
+            line = f'baseurl={C}/{prefix}/{p} {origin}{p}'
+            state['hit'] = True; state['sig'] = state['sig'] or sig
+            break
+    sec.append(line)
+flush()
+open(f, 'w').write('\n'.join(out))
+PY
+}
+
+case "$MODE" in
+  apply)
+    C="${2:?usage: apply https://<cache>}"; C="${C%/}"
+    case "$C" in https://*) ;; *) die "cache URL must be https://" ;; esac
+    [ -s "$CA_SRC" ] || die "missing $CA_SRC (copy keys/pkg-cache/ca.crt next to this script)"
+    B=/root/pkgcache-repo-backup-$(date -u +%Y%m%dT%H%M%SZ)
+    mkdir -p "$B/yum.repos.d"
+    cp -a /etc/yum.repos.d/*.repo "$B/yum.repos.d/"
+    [ -e "$ANCHOR" ] && cp -a "$ANCHOR" "$B/" || touch "$B/.no-anchor-before"
+    install -m 0644 "$CA_SRC" "$ANCHOR" && update-ca-trust
+    for f in /etc/yum.repos.d/almalinux*.repo /etc/yum.repos.d/epel*.repo /etc/yum.repos.d/remi*.repo; do
+        [ -f "$f" ] || continue
+        grep -q -E '^#?\s*baseurl\s*=\s*https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" && rewrite "$f" "$C"
+    done
+    echo "backup: $B"
+    echo "changed files:"; for f in "$B"/yum.repos.d/*.repo; do same "$f" "/etc/yum.repos.d/$(basename "$f")" || echo "  /etc/yum.repos.d/$(basename "$f")"; done
+    ;;
+  revert)
+    B="${2:-$(ls -d /root/pkgcache-repo-backup-* 2>/dev/null | sort | tail -1)}"
+    [ -d "$B/yum.repos.d" ] || die "no backup dir ($B)"
+    for f in "$B"/yum.repos.d/*.repo; do same "$f" "/etc/yum.repos.d/$(basename "$f")" || { cp -a "$f" /etc/yum.repos.d/; echo "restored /etc/yum.repos.d/$(basename "$f")"; }; done
+    if [ -e "$B/.no-anchor-before" ]; then rm -f "$ANCHOR"; update-ca-trust; echo "removed $ANCHOR"; fi
+    dnf clean metadata -q || true
+    echo "reverted from $B"
+    ;;
+  status)
+    grep -l -E '^baseurl=https://[^ ]+/(almalinux|epel|remi)/' /etc/yum.repos.d/*.repo 2>/dev/null | sed 's/^/via cache: /' || echo "no repo uses the cache"
+    ls -d /root/pkgcache-repo-backup-* 2>/dev/null | sed 's/^/backup: /' || true
+    [ -e "$ANCHOR" ] && echo "CA anchor: $ANCHOR" || echo "CA anchor: absent"
+    ;;
+  *) die "usage: apply URL | status | revert [DIR]" ;;
+esac

scripts/pkg-cache-repos.sh

@@ -0,0 +1,223 @@
+#!/bin/bash
+## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal
+## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster)
+## when PKG_CACHE is set; without it the repo files are left alone and builds
+## use the upstream repositories as before.
+##
+##   PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm      # AlmaLinux/EPEL/Remi
+##   PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt      # Ubuntu + LiteSpeed
+##   pkg-cache-repos.sh release-install epel [remi]             # GPG-verified release RPMs
+##   pkg-cache-repos.sh restore                                 # undo everything above
+##
+## Every RUN that rewrites must `restore` before it ends: it puts rewritten files
+## back, deletes files this script created, deletes any NEW repo/apt file that
+## still references the cache, then FAILS if any reference is left — so no shipped
+## layer carries the cache's LAN address (images run on hosts that can't reach it).
+##
+## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging
+## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror
+## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with
+## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and
+## immutable .rpm/.deb files long-term.
+##
+## Integrity (security review 2026-10-21):
+##  * PKG_CACHE must be https://. The cache's certificate is issued by the
+##    name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only
+##    for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten
+##    repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN —
+##    never added to the system trust store.
+##  * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and
+##    AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL
+##    publishes no repomd signature (404), so its metadata relies on HTTPS end
+##    to end (client->cache->dl.fedoraproject.org/mirrors, all verified).
+##  * release-install downloads epel-release / remi-release (from the cache, or
+##    from the HTTPS origin without PKG_CACHE), checks the package name, imports
+##    the signing keys committed in keys/pkg-cache/ and requires
+##    `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1.
+##  * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail
+##    a build that gets old PHP (replay of old-but-signed metadata).
+##
+## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if
+##      the cache dies mid-build dnf fails over to the origin by itself.
+## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the
+##      cache (with the CA), and a build without PKG_CACHE uses upstream sources.
+set -euo pipefail
+
+MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|restore}"
+shift || true
+STATE=/var/lib/pkg-cache-repos          # saved originals + manifest + created files
+SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}"   # ca.crt + RPM-GPG-KEY-*
+C="${PKG_CACHE:-}"
+C="${C%/}"
+
+die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; }
+
+if [ -n "$C" ]; then
+    case "$C" in
+        https://*) ;;
+        *) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;;
+    esac
+    HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}"
+    [ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing"
+fi
+
+# Files that may hold repo configuration; restore re-checks all of them.
+repo_files() {
+    find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true
+}
+
+start_session() {
+    [ -d "$STATE" ] && return 0
+    mkdir -p "$STATE/orig"
+    repo_files > "$STATE/manifest"     # what existed before this RUN touched anything
+    : > "$STATE/created"
+    printf '%s\n' "$HOSTPORT" > "$STATE/host"
+}
+save() {                                # keep the first-seen (pre-rewrite) copy
+    [ -e "$STATE/orig$1" ] && return 0
+    mkdir -p "$STATE/orig$(dirname "$1")"
+    cp -p "$1" "$STATE/orig$1"
+}
+created() { printf '%s\n' "$1" >> "$STATE/created"; }
+
+if [ "$MODE" = restore ]; then
+    [ -d "$STATE" ] || exit 0
+    host=$(cat "$STATE/host")
+    (cd "$STATE/orig" && find . -type f) | while read -r rel; do
+        cat "$STATE/orig/$rel" > "/${rel#./}"
+    done
+    while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created"
+    # A repo/apt file that appeared during this RUN and still names the cache
+    # (e.g. written by a package from the rewritten repos) is removed too.
+    repo_files | while read -r f; do
+        if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then
+            echo "pkg-cache-repos: removing new file that references the cache: $f"
+            rm -f -- "$f"
+        fi
+    done
+    left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true)
+    rm -rf "$STATE"
+    [ -z "$left" ] || die "cache address $host still present after restore in: $left"
+    echo "pkg-cache-repos: original repository files restored (no reference to $host left)"
+    exit 0
+fi
+
+if [ "$MODE" = release-install ]; then
+    [ $# -gt 0 ] || die "release-install needs: epel and/or remi"
+    tmp=$(mktemp -d)
+    curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120)
+    if [ -n "$C" ]; then
+        start_session
+        cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"
+        curl_opts+=(--cacert "$STATE/ca-bundle.pem")
+    fi
+    files=()
+    for name in "$@"; do
+        case "$name" in
+            epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-10
+                  path=/epel/epel-release-latest-10.noarch.rpm
+                  origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm ;;
+            remi) pkg=remi-release key=RPM-GPG-KEY-remi2024
+                  path=/remi/enterprise/remi-release-10.rpm
+                  origin=https://rpms.remirepo.net/enterprise/remi-release-10.rpm ;;
+            *) die "unknown release package '$name'" ;;
+        esac
+        url="$origin"; [ -n "$C" ] && url="$C$path"
+        f="$tmp/$pkg.rpm"
+        curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url"
+        rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key"
+        got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true)
+        [ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg"
+        chk=$(rpm -K "$f" 2>&1 || true)
+        [[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk"
+        echo "pkg-cache-repos: $pkg verified ($chk) from $url"
+        files+=("$f")
+    done
+    dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}"
+    rm -rf "$tmp"
+    exit 0
+fi
+
+if [ -z "$C" ]; then
+    echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories"
+    exit 0
+fi
+
+rewrite_rpm() {
+    local f="$1"
+    grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" || return 0
+    save "$f"
+    # Per [section]: a (commented or live) baseurl on a known origin becomes
+    # "baseurl=<cache>/<prefix>/<path> <origin>/<path>"; that section's
+    # mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it
+    # gets sslcacert=<system bundle + pkg-cache CA>, gpgcheck=1 and, for repos
+    # that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched.
+    awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" '
+        function flush(   i) {
+            for (i = 1; i <= n; i++) {
+                if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i]
+                if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue
+                print buf[i]
+            }
+            if (hit) {
+                print "sslcacert=" BUNDLE
+                print "gpgcheck=1"
+                if (signed) print "repo_gpgcheck=1"
+            }
+            n = 0; hit = 0; signed = 0
+        }
+        function rw(line, re, prefix, origin, sig,   p) {
+            if (match(line, re)) {
+                p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p)
+                hit = 1; if (sig) signed = 1
+                return "baseurl=" C "/" prefix "/" p " " origin "/" p
+            }
+            return line
+        }
+        /^\[/ { flush() }
+        {
+            l = $0
+            if (l ~ /^#? ?baseurl=/) {
+                l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1)
+                l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0)
+                l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1)
+            }
+            buf[++n] = l
+        }
+        END { flush() }
+    ' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache"
+}
+
+rewrite_apt() {
+    local f="$1"
+    grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/' "$f" || return 0
+    save "$f"
+    sed -i -E \
+        -e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \
+        -e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \
+        -e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \
+        "$f"
+}
+
+case "$MODE" in
+    rpm)
+        start_session
+        cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"
+        for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done
+        echo "pkg-cache-repos(rpm): repos now via $C:"
+        grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/  /'
+        ;;
+    apt)
+        start_session
+        conf=/etc/apt/apt.conf.d/99pkg-cache
+        # Trust the pkg-cache CA for the cache host ONLY (apt per-host option).
+        printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf"
+        created "$conf"
+        for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
+            [ -f "$f" ] && rewrite_apt "$f"
+        done
+        echo "pkg-cache-repos(apt): sources now via $C:"
+        grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | grep -F "$C/" | sort -u | sed 's/^/  /'
+        ;;
+    *) die "unknown mode '$MODE'" ;;
+esac