cloud-hosting-platform/cloud-apache-container
6
0 Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(pkg-cache): HTTPS-only cache, GPG-verified release RPMs, strict restore

Browse Source 
Security review follow-up for the opt-in PKG_CACHE path:

- PKG_CACHE must be https:// (http is refused). The cache's certificate
  comes from a name-constrained internal CA (keys/pkg-cache/ca.crt, valid
  only for 192.168.1.40 / pkg-cache.*). The CA is trusted only by the
  rewritten repos (dnf sslcacert bundle, apt per-host CaInfo) and only
  for the duration of the RUN.
- epel-release / remi-release are no longer installed straight from a
  URL. release-install downloads them over HTTPS, checks the package name,
  imports the signing keys committed in keys/pkg-cache/ and requires
  `rpm -K` "signatures OK", then installs with localpkg_gpgcheck=1. This
  applies with and without the cache.
- Rewritten repos get gpgcheck=1. Remi and AlmaLinux also get
  repo_gpgcheck=1, since both sign repomd.xml. EPEL does not publish a
  repomd signature, so it relies on end-to-end HTTPS.
- restore also deletes files it created and any new repo/apt file that
  still references the cache, then fails the build if a reference remains
  under /etc/yum.repos.d, /etc/dnf or /etc/apt.
- docs: trust model, replay risk, proposed openssl/httpd/ImageMagick
  minimum-version gates, TLS registry mirror (BuildKit ca= / dockerd
  certs.d, no insecure-registries).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-10-21 12:53:24 -07:00
1 parent b2bec5b3ee
commit 18f65c6d84
10 files changed
+411 -134
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Dockerfile
+5 -3
View File
@@ -7,15 +7,17 @@ ARG CACHE_BUST=
# php.net's current release for this branch (CI); the build fails on older PHP.
ARG EXPECT_PHP_MIN=
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40
# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
# GPG-verified against keys/pkg-cache/ either way. Repo files
# are restored at the end of every RUN, so no layer keeps the cache address.
# See scripts/pkg-cache-repos.sh.
ARG PKG_CACHE=
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
# Install repos, update, install only needed packages, clean up in one layer
RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \
/usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \
# Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security
# release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out),
# so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback
Dockerfile.fpm
+5 -3
View File
@@ -7,15 +7,17 @@ ARG CACHE_BUST=
# php.net's current release for this branch (CI); the build fails on older PHP.
ARG EXPECT_PHP_MIN=
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40
# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
# GPG-verified against keys/pkg-cache/ either way. Repo files
# are restored at the end of every RUN, so no layer keeps the cache address.
# See scripts/pkg-cache-repos.sh.
ARG PKG_CACHE=
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
# Install repos, update, install only needed packages (no httpd/mod_ssl), clean up in one layer
RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls) && \
/usr/local/sbin/pkg-cache-repos.sh release-install epel remi && \
# Use Remi's origin, not the mirrorlist: during the 2026-10-21 PHP security
# release the mirrors lagged hours behind (still 8.2.34 when 8.2.35 was out),
# so builds shipped unpatched PHP. HTTPS first; plain http only as a fallback
Dockerfile.shared-httpd
+5 -3
View File
@@ -1,15 +1,17 @@
FROM almalinux/10-base
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40
# (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
# (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
# GPG-verified against keys/pkg-cache/ either way. Repo files
# are restored at the end of every RUN, so no layer keeps the cache address.
# See scripts/pkg-cache-repos.sh.
ARG PKG_CACHE=
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
# Install Apache and minimal dependencies (no PHP at all)
RUN /usr/local/sbin/pkg-cache-repos.sh rpm && \
dnf install -y $(/usr/local/sbin/pkg-cache-repos.sh release-urls | cut -d' ' -f1) && \
/usr/local/sbin/pkg-cache-repos.sh release-install epel && \
/usr/local/sbin/pkg-cache-repos.sh rpm && \
dnf update -y && \
dnf install -y httpd mod_ssl openssl iproute cronie procps curl && \
Dockerfile.shared-ols
+4 -2
View File
@@ -19,12 +19,14 @@ ARG OLS_VERSION=1.8.4
ARG PHPVER=83
FROM litespeedtech/openlitespeed:${OLS_VERSION}-lsphp${PHPVER}
## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=http://192.168.1.40
## (CT130 pkg-cache). Unset = upstream repositories exactly as before. Repo files
## OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
## (CT130 pkg-cache, https only). Unset = upstream repositories. Release RPMs are
## GPG-verified against keys/pkg-cache/ either way. Repo files
## are restored at the end of every RUN, so no layer keeps the cache address.
## See scripts/pkg-cache-repos.sh.
ARG PKG_CACHE=
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
## Tooling the shared tier needs on top of the base:
## - inotify-tools: the .htaccess watcher (spec 5.3)
docs/pkg-cache.md
+135 -69
View File
@@ -1,89 +1,128 @@
# Internal package + image cache (`PKG_CACHE`)
Opt-in build support for the LAN package cache **CT130 `pkg-cache` = `192.168.1.40`**
(Proxmox node `pve`, storage `shared-storage-tn`). Without `PKG_CACHE`, builds use
the upstream repositories exactly as before.
Opt-in build support for the LAN package cache **CT130 `pkg-cache` at `https://192.168.1.40`** (Proxmox node `pve`, storage `shared-storage-tn`).
Why it exists: during the 2026-10-21 PHP security release, builds failed because
Remi mirrors lagged hours behind origin, Remi's origin reset or timed out HTTPS
under load, and an EPEL mirror had a broken certificate.
Without `PKG_CACHE`, builds use the upstream repositories. The one change from before is that the EPEL and Remi release RPMs are now **GPG-verified** against `keys/pkg-cache/` before they are installed.
Why it exists: during the 2026-10-21 PHP security release, builds failed because:
- Remi mirrors lagged hours behind origin;
- Remi's origin reset or timed out HTTPS under load;
- an EPEL mirror had a broken certificate.
On that day a build without the cache spent **1031 s** in its first dnf step. Through the cache the same step took **40 s cold and 21 s warm**.
## What the cache serves
| URL on the cache | Upstream tiers, tried in order (stale copy last) |
| URL (https, pkg-cache CA) | Upstream tiers, tried in order (stale copy last) |
|---|---|
| `http://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) |
| `http://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` |
| `http://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` |
| `http://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` |
| `http://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` |
| `http://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` |
| `192.168.1.40:5000` (plain http) | Docker Hub pull-through cache (`registry-1.docker.io`) |
| `https://192.168.1.40/remi/` | `https://rpms.remirepo.net`, then `http://rpms.remirepo.net` (origin only, no mirrors) |
| `https://192.168.1.40/epel/` | `https://dl.fedoraproject.org/pub/epel`, `https://mirror.math.princeton.edu/pub/epel`, `https://mirrors.kernel.org/fedora-epel` |
| `https://192.168.1.40/almalinux/` | `https://repo.almalinux.org/almalinux`, `https://mirror.math.princeton.edu/pub/almalinux` |
| `https://192.168.1.40/ubuntu/` | `https://archive.ubuntu.com/ubuntu`, `https://us.archive.ubuntu.com/ubuntu` |
| `https://192.168.1.40/ubuntu-security/` | `https://security.ubuntu.com/ubuntu`, `https://archive.ubuntu.com/ubuntu` |
| `https://192.168.1.40/litespeed/` | `https://rpms.litespeedtech.com`, then `http://rpms.litespeedtech.com` |
| `192.168.1.40:5000` (TLS, pkg-cache CA) | Docker Hub pull-through cache (`registry-1.docker.io`) |
Plain `http://192.168.1.40/` answers **403** to LAN clients.
Cache rules:
- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata here means `repomd.xml*`, repodata files whose names don't start with a hash, apt `dists/` except `by-hash/`, `*-latest*.rpm`, `remi-release-N.rpm`, `RPM-GPG-KEY*` and directory listings.
- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are kept for 30 days, and evicted after 60 days without use. The cache is capped at 110 GB (`min_free` 15 GB).
- On a connect error, timeout or 5xx, a request falls through to the next tier. Only the **last** tier answers from a stale copy, so metadata goes primary → fallback → stale. A fresher fallback always wins over stale.
- `proxy_cache_lock` is on: concurrent misses on one file make a single upstream fetch.
- Each response carries `X-Cache-Status` (`HIT`/`MISS`/`REVALIDATED`/`EXPIRED`/`STALE`/`BYPASS`) and `X-Pkgcache-Tier`. On a HIT, the tier header names the location, not the upstream that filled the entry.
- **Metadata** has a 3-minute TTL. After that it is revalidated upstream (`If-Modified-Since`). Metadata means:
- `repomd.xml*`
- repodata files whose names don't start with a hash
- apt `dists/`, except `by-hash/`
- `*-latest*.rpm` and `remi-release-N.rpm`
- `RPM-GPG-KEY*`
- directory listings
- **Packages** (`*.rpm`, `*.deb`, hash-named repodata, `by-hash/`) are cached for 30 days and evicted after 60 days unused. Total size is capped at 110 GB, with 15 GB kept free.
- On a connect error, timeout or 5xx, the request falls through to the next tier. Only the **last** tier may answer from a stale copy, so the order is primary, then fallback, then stale.
- `proxy_cache_lock` is on.
- Responses carry `X-Cache-Status` and `X-Pkgcache-Tier`.
## Building with the cache
```bash
docker buildx build --build-arg PKG_CACHE=http://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile .
docker buildx build --build-arg PKG_CACHE=https://192.168.1.40 --build-arg PHPVER=85 -f Dockerfile .
```
`scripts/pkg-cache-repos.sh` (copied to `/usr/local/sbin/`) is wired into
`Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`:
`scripts/pkg-cache-repos.sh` (installed as `/usr/local/sbin/`) and `keys/pkg-cache/` (installed as `/usr/local/share/pkg-cache/`) are wired into `Dockerfile`, `Dockerfile.fpm`, `Dockerfile.shared-httpd` and `Dockerfile.shared-ols`. The script has four modes.
- `rpm`: rewrites the AlmaLinux, EPEL and Remi `baseurl`s to `<cache>/<prefix>/... <origin>/...`, and comments out that section's mirrorlist/metalink. dnf treats the second URL as a failover, so a dead cache falls back to the origin by itself.
- `apt`: rewrites the Ubuntu archive/security and LiteSpeed URIs to the cache. apt has no failover, so CI must probe the cache first (see below).
- `release-urls`: the EPEL and Remi release RPM URLs, from the cache when it is set.
- `restore`: puts the original repo files back. Every RUN that rewrites ends with `restore`, so **no shipped layer contains the cache address**. Images run on datacenter hosts that cannot reach it.
**`rpm`** handles AlmaLinux, EPEL and Remi:
- Each `baseurl` becomes `<cache>/<prefix>/... <origin>/...`. dnf fails over to the origin if the cache dies.
- That section's mirrorlist/metalink is commented out.
- The section gets `sslcacert=` pointing at a bundle of the system CAs plus the pkg-cache CA. The bundle lives in the state dir and is deleted by `restore`.
- It also gets `gpgcheck=1`.
- AlmaLinux and Remi also get `repo_gpgcheck=1`, because both sign `repomd.xml` (measured 2026-10-21: `repomd.xml.asc` returns 200). EPEL publishes no repomd signature (404), so its metadata integrity rests on HTTPS end to end.
`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Their
multi-stage `lsphp-apt-upgrade.sh` flow needs two changes first:
- a `PKG_CACHE` ARG in each stage;
- the madison check accepting the cache URL. It currently requires `*rpms.litespeedtech.com*`, and through the cache `apt-cache madison` shows `http://192.168.1.40/litespeed/debian`.
**`apt`** handles Ubuntu archive/security and LiteSpeed:
- Their URIs are rewritten to the cache.
- `/etc/apt/apt.conf.d/99pkg-cache` sets `Acquire::https::192.168.1.40::CaInfo`, so the CA is trusted for the cache host only.
- apt verifies the signed `InRelease`/`Release.gpg` as always. It has no failover, so CI must probe the cache first.
The PHP gates are unchanged and still apply: `EXPECT_PHP_MIN`,
`EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`.
**`release-install epel [remi]`**:
- Downloads the release RPM over HTTPS, from the cache or from the origin.
- Checks the package name.
- Imports the key committed in `keys/pkg-cache/`.
- Requires `rpm -K` to report `digests signatures OK`.
- Installs with `--setopt=localpkg_gpgcheck=1`.
## Why plain HTTP on the LAN (trade-off)
**`restore`**:
- Puts rewritten files back.
- Deletes files the script created.
- Deletes any repo/apt file that **appeared during the RUN** and still references the cache.
- Then **fails the build** if any file under `/etc/yum.repos.d`, `/etc/dnf` or `/etc/apt` still mentions the cache.
- The cache fetches upstream over **HTTPS with certificate verification**. Remi and LiteSpeed also have a plain-http origin fallback, which mirrors what the Dockerfile already did.
- Client to cache is plain HTTP. dnf still verifies **package GPG signatures**, and apt verifies the signed `InRelease`/`Release.gpg` plus package hashes. A LAN attacker therefore cannot inject packages. On 2026-10-21 apt measurably refused the cached LiteSpeed index when given the wrong key (`NO_PUBKEY 011AA62DEDA1F085`).
- The residual risk is **metadata replay or freeze**: an attacker on the LAN path, or on the plain-http fallback leg, serves an *old* but validly signed `repomd.xml`. That would let a build install old PHP. Remi/EPEL/AlmaLinux do not enable `repo_gpgcheck`, so dnf does not check metadata signatures. The **php.net version gates** cover this for our images: a build fails if it gets PHP older than php.net's current release.
- WHP **hosts** have no such gate. They should keep the origin as the second `baseurl` and not rely on the cache for freshness on security-release day. Run `pkgcache-refresh` on the cache instead.
- Signing keys must **not** be fetched through the cache. Take them from the base image or directly over HTTPS.
- HTTPS to the cache (internal CA or a Let's Encrypt DNS-01 certificate for `pkg-cache.internal.cloud-hosting.io`) would close the LAN leg. It needs a CA in every build image or a public DNS name plus certificate. Not done; Josh to decide.
Every RUN that rewrites ends with `restore`, so no shipped layer carries the LAN address. This was verified on the built images: 0 references, no state dir, no apt drop-in.
`PKG_CACHE` must start with `https://`. `http://` is refused.
`Dockerfile.litespeed` and `Dockerfile.lsphp` are **not wired yet**. Two things are needed first:
- the multi-stage `lsphp-apt-upgrade.sh` flow needs a `PKG_CACHE` ARG in each stage;
- its madison check must accept the cache URL. It currently requires `*rpms.litespeedtech.com*`, and via the cache madison shows `https://192.168.1.40/litespeed/debian`.
## Trust model
`keys/pkg-cache/ca.crt` is the *pkg-cache internal CA*:
- EC P-256, valid to 2036.
- **Name-constrained** to IP `192.168.1.40` and DNS `pkg-cache.internal.cloud-hosting.io` / `pkg-cache.geekhome.local`. Even its private key, which exists only on CT130 at `/etc/pkgcache-ca/ca.key`, cannot mint a certificate that clients would accept for any other host.
- It is trusted only by the rewritten repos, the apt per-host option and the BuildKit/dockerd registry config. It is never added to a system trust store.
The server certificate `/etc/pkgcache-ca/server.crt` **expires 2029-10-20**. `pkgcache-stats` prints the expiry. Renew it with the same `openssl x509 -req ... -extfile srv.cnf` steps in `/etc/pkgcache-ca/`.
Remaining risks:
- **Replay of old but validly signed metadata.** Signatures stop tampering, not freshness. The cache's own plain-http tiers for Remi and LiteSpeed are the leg where an internet MITM could replay. The php.net gates (`EXPECT_PHP_MIN`, `EXPECT_LSPHP_PKG`/`EXPECT_PHP_VERSION`) fail an image build that gets old PHP. **Josh to decide** whether to drop those http tiers: you gain replay resistance and lose availability when Remi's HTTPS is down, as it was all day on 2026-10-21.
- **Proposed, not implemented:** the same kind of minimum-version gate for other security-relevant packages:
- CI resolves the newest `openssl`/`openssl-libs` and `httpd` from AlmaLinux BaseOS/AppStream, and `ImageMagick` from EPEL, **directly from the origin over HTTPS** (e.g. `dnf repoquery --repofrompath=o,https://repo.almalinux.org/almalinux/10/BaseOS/x86_64/os/ --repo=o --latest-limit=1 --qf '%{evr}' openssl-libs`).
- It passes the results as `EXPECT_OPENSSL_MIN` / `EXPECT_HTTPD_MIN` / `EXPECT_IMAGEMAGICK_MIN`.
- The Dockerfile asserts `rpm -q --qf '%{EVR}'` is not older (`rpm --eval "%{lua: print(rpm.vercmp('A','B'))}"`).
## CI integration plan (NOT applied — needs Josh)
1. **Network (blocker).** The ephemeral runners (`eph-pve-*` on `pve` vmbr2 10.99.98.0/24, `eph-pve3-*` on `pve3` vmbr1 10.99.99.0/24) are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40. A narrow exception is needed on each node, placed before the DROP lines in `/etc/network/interfaces.d/*`:
1. **Network (blocker).** The ephemeral runners are NATed, and `iptables FORWARD` **drops all RFC1918 destinations**, including 192.168.1.40:
- `eph-pve-*` on `pve`, vmbr2, 10.99.98.0/24;
- `eph-pve3-*` on `pve3`, vmbr1, 10.99.99.0/24.
A narrow exception is needed, placed before the DROP lines:
```
# pve (/etc/network/interfaces.d/ vmbr2 stanza), before the 192.168.0.0/16 DROP:
post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT
post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT
# live, without ifreload:
iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 80,5000 -j ACCEPT
# pve3: same with vmbr1
# pve, /etc/network/interfaces.d/ vmbr2 stanza, BEFORE the 192.168.0.0/16 DROP:
post-up iptables -A FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT
post-down iptables -D FORWARD -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT
# live: iptables -I FORWARD 1 -i vmbr2 -o vmbr0 -d 192.168.1.40 -p tcp -m multiport --dports 443,5000 -j ACCEPT
# pve3: the same with vmbr1
```
`gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach 192.168.1.40 is untested.
2. **Probe, then opt in** (no `${{ }}` inside `run:`, per repo rule):
`gitea-runner.internal.cloud-hosting.io` resolves to 10.10.14.20, which is not on this LAN. Whether it can reach the cache is untested.
2. **Probe with the CA, then opt in.** Keep `${{ }}` out of `run:`. The probe fails when the cache is down **or its certificate is expired or untrusted**, and the build then uses upstream.
```yaml
- name: Probe package cache
id: pkgcache
run: |
if curl -fsS --max-time 5 -o /dev/null http://192.168.1.40/ ; then
echo "url=http://192.168.1.40" >> "$GITHUB_OUTPUT"
if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/ ; then
echo "url=https://192.168.1.40" >> "$GITHUB_OUTPUT"
else
echo "url=" >> "$GITHUB_OUTPUT"; echo "::warning::pkg-cache unreachable - building from upstream"
fi
# in docker/build-push-action `build-args:` add
# PKG_CACHE=${{ steps.pkgcache.outputs.url }}
# docker/build-push-action `build-args:` add PKG_CACHE=${{ steps.pkgcache.outputs.url }}
```
3. **Docker Hub mirror for BuildKit.** The workflows use `docker/setup-buildx-action` (docker-container driver). BuildKit **ignores dockerd's `registry-mirrors`**, so configure the builder instead:
3. **Docker Hub mirror for BuildKit**, over TLS and without `insecure`. `setup-buildx-action` uses the docker-container driver, which **ignores dockerd's `registry-mirrors`**. buildx copies the `ca` file into the builder.
```yaml
- uses: docker/setup-buildx-action@v3
with:
@@ -91,34 +130,61 @@ The PHP gates are unchanged and still apply: `EXPECT_PHP_MIN`,
[registry."docker.io"]
mirrors = ["192.168.1.40:5000"]
[registry."192.168.1.40:5000"]
http = true
ca = ["keys/pkg-cache/ca.crt"]
```
This exact config was used for the test builds on 192.168.1.193. Only Docker Hub images go through it; `repo.anhonesthost.net/...` is unaffected.
4. **dockerd clients** (plain `docker pull`, e.g. the build server or test hosts). Untested, because applying it needs a daemon restart. Add to `/etc/docker/daemon.json` and restart docker:
```json
{ "registry-mirrors": ["http://192.168.1.40:5000"], "insecure-registries": ["192.168.1.40:5000"] }
The exact equivalent (absolute `ca` path) was measured on 192.168.1.193. BuildKit falls back to Docker Hub when the mirror is down; this was measured too, with the registry stopped. The relative `ca` path inside a workflow is **untested**; use `${{ github.workspace }}/keys/pkg-cache/ca.crt` if it doesn't resolve.
4. **dockerd clients** (plain `docker pull`). Untested, because it needs a daemon restart. Use TLS, no `insecure-registries`:
```
install -D -m0644 keys/pkg-cache/ca.crt /etc/docker/certs.d/192.168.1.40:5000/ca.crt
# /etc/docker/daemon.json: { "registry-mirrors": ["https://192.168.1.40:5000"] } then restart docker
```
## WHP hosts (plan only)
The production hosts (whp01/whp02/sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts such as 192.168.1.148 can. Options for Josh:
- (a) a second identical cache CT in the datacenter;
The production hosts (whp01, whp02, sdbees) are in the datacenter and **cannot reach 192.168.1.40**. Only LAN hosts can, e.g. 192.168.1.148. Options for Josh:
- (a) a second identical cache in the datacenter, with its own name-constrained CA;
- (b) expose this one over NetBird;
- (c) LAN hosts only.
For a host that can reach a cache, use the same idea as `pkg-cache-repos.sh rpm`, keeping the origin second so the cache is never a single point of failure:
For a host that can reach a cache:
- install the CA at `/etc/pki/pkg-cache/ca.crt`;
- create a bundle (`cat /etc/pki/tls/certs/ca-bundle.crt /etc/pki/pkg-cache/ca.crt > /etc/pki/pkg-cache/bundle.pem`);
- in each repo section, keep the origin as the second URL:
```ini
# /etc/yum.repos.d/remi-safe.repo [remi-safe] (AL9: enterprise/9)
baseurl=http://<cache>/remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/
[remi-safe]
baseurl=https://<cache>/remi/enterprise/$releasever_major/safe/$basearch/ https://rpms.remirepo.net/enterprise/$releasever_major/safe/$basearch/
#mirrorlist=...
sslcacert=/etc/pki/pkg-cache/bundle.pem
gpgcheck=1
repo_gpgcheck=1 # Remi + AlmaLinux sign repomd; EPEL does not
```
Hosts have no php.net gate. On a security-release day, run `pkgcache-refresh` on the cache and check `dnf info php-cli` against php.net.
## Operating the cache (CT130)
- Config: `/etc/nginx/conf.d/pkgcache-http.conf` (cache zone, maps, log format) and `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf`. `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`: edit its `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`.
- Registry: `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`), service `docker-registry`.
- Data: nginx in `/srv/cache/nginx`, registry in `/srv/cache/registry` (150 GB mount point `mp0`, `backup=0`).
- **Security-release day:** run `pkgcache-refresh` to re-fetch every cached metadata entry now (or `pkgcache-refresh remi` for one prefix; `--list` shows them; `--purge` deletes them instead). A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Without it, metadata is at most 3 minutes old anyway.
- Monitoring: `pkgcache-stats` shows disk, hit ratio, fallback-tier use, STALE count and upstream errors. Logs are in `/var/log/nginx/pkgcache.access.log` (with `cache=`, `tier=`, `up=` fields) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks: `http://192.168.1.40/` and `http://192.168.1.40:5000/v2/`.
- Docker Hub: anonymous pulls are rate-limited per egress IP, and every client now shares this cache's IP. To raise the limit, add `username`/`password` (a Hub access token) under `proxy:`.
- **nginx config.**
- `/etc/nginx/conf.d/pkgcache-http.conf` holds the cache zone, maps and log format.
- `/etc/nginx/pkgcache/{server-common,meta,pkg}.conf` hold the shared settings.
- `/etc/nginx/sites-available/pkgcache` is **generated** by `/usr/local/sbin/pkgcache-gen-nginx`. Edit the `UPSTREAMS` table, then run `pkgcache-gen-nginx > /etc/nginx/sites-available/pkgcache && nginx -t && systemctl reload nginx`.
- Listeners: :443 TLS for the LAN, `127.0.0.1:80` for local tools, and :80 for the LAN answers 403.
- **Registry.** Config is `/etc/docker/registry/config.yml` (Debian `docker-registry` 2.8.3, the same code as `registry:2`, with TLS). Service `docker-registry`.
- **TLS.** Everything is in `/etc/pkgcache-ca/`: `ca.key` (0600), `ca.crt`, `server.key`, `server-chain.crt`, and the `ca.cnf`/`srv.cnf` openssl configs.
- **Data.** nginx uses `/srv/cache/nginx`, the registry uses `/srv/cache/registry`. Both live on the 150 GB mount point `mp0` (`backup=0`).
- **Security-release day.** `pkgcache-refresh` re-fetches every cached metadata entry right away. Variants:
- `pkgcache-refresh remi` refreshes one prefix;
- `--list` lists the entries;
- `--purge` deletes them instead.
A refresh bypasses the cache from localhost only and replaces each entry in place. If upstream is down, the old copy stays. Metadata is at most 3 minutes old anyway.
- **Monitoring.** `pkgcache-stats` shows:
- disk usage;
- hit ratio;
- fallback-tier use;
- the STALE count;
- upstream errors;
- certificate expiry.
Logs are `/var/log/nginx/pkgcache.access.log` (fields `cache=`, `tier=`, `up=`) and `journalctl -u docker-registry`. Suggested Uptime Kuma checks are `https://192.168.1.40/` and `https://192.168.1.40:5000/v2/`, with certificate checking against the CA.
- **Docker Hub rate limits.** Anonymous pulls are limited per egress IP, and all clients share it. Add `username`/`password` (a Hub access token) under `proxy:` if needed.
keys/pkg-cache/README
+20
View File
@@ -0,0 +1,20 @@
Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh,
docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing
here is added to the system trust store.
ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18.
NAME-CONSTRAINED to IP 192.168.1.40 and DNS
pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local.
SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3
(private key lives only on CT130: /etc/pkgcache-ca/ca.key)
RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158
fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and
mirror.math.princeton.edu (identical); fingerprint matches
https://fedoraproject.org/security/
RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4
fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical
to the key shipped in remi-release-10.2-1.el10.remi. Not found on
keys.openpgp.org / keyserver.ubuntu.com - confirm out of band.
Remi rotates keys (remi2025, ...): when remi-release starts being
signed by a new key, `release-install remi` fails loudly - add the
new key here (and to pkg-cache-repos.sh).
keys/pkg-cache/RPM-GPG-KEY-EPEL-10
+29
View File
@@ -0,0 +1,29 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA
Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK
Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L
URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf
mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2
XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI
uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60
GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix
fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z
BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77
HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB
tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB
CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC
BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p
l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp
yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg
Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u
scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs
7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK
F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg
sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB
0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY
1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt
TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k
p9Dfd2pKuxK9Mys=
=mhQZ
-----END PGP PUBLIC KEY BLOCK-----
keys/pkg-cache/RPM-GPG-KEY-remi2024
+52
View File
@@ -0,0 +1,52 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W
Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe
UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa
qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN
ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB
jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO
/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL
yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ
w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN
0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l
ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB
tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly
ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd
/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX
gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx
Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W
4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/
/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz
/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc
3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof
ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS
dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy
sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n
i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd
DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P
7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI
T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ
OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3
cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W
RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x
5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m
8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD
pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r
N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm
Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG
Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA
AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9
OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI
wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA
qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c
WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV
u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI
qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S
4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4
MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II
Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl
bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s
r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1
=meaU
-----END PGP PUBLIC KEY BLOCK-----
keys/pkg-cache/ca.crt
+15
View File
@@ -0,0 +1,15 @@
-----BEGIN CERTIFICATE-----
MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw
TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV
BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN
MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg
KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG
ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV
OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD
VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/
v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw
a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu
Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg
cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM
YAIl
-----END CERTIFICATE-----
scripts/pkg-cache-repos.sh
+141 -54
View File
@@ -1,63 +1,140 @@
#!/bin/bash
## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal
## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster)
## when PKG_CACHE is set; a no-op otherwise, so builds behave exactly as before
## without it.
## when PKG_CACHE is set; without it the repo files are left alone and builds
## use the upstream repositories as before.
##
## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi
## PKG_CACHE=http://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu + LiteSpeed
## pkg-cache-repos.sh release-urls # prints EPEL+Remi release RPM URLs
## pkg-cache-repos.sh restore # put the original repo files back
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu + LiteSpeed
## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs
## pkg-cache-repos.sh restore # undo everything above
##
## Every RUN that rewrites must `restore` before it ends, so no shipped layer
## carries the cache's LAN address in /etc/yum.repos.d or /etc/apt (images run on
## hosts that cannot reach it). restore is a no-op when nothing was rewritten.
## Every RUN that rewrites must `restore` before it ends: it puts rewritten files
## back, deletes files this script created, deletes any NEW repo/apt file that
## still references the cache, then FAILS if any reference is left — so no shipped
## layer carries the cache's LAN address (images run on hosts that can't reach it).
##
## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging
## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror
## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with
## origin/mirror fallbacks and stale-on-error), keeps metadata only ~3 minutes,
## and keeps immutable .rpm/.deb files long-term.
## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and
## immutable .rpm/.deb files long-term.
##
## Integrity is unchanged: dnf/apt still verify package GPG signatures (and apt
## the signed InRelease/Release.gpg); the build's php.net version gates
## (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail a build that gets old PHP.
## Integrity (security review 2026-10-21):
## * PKG_CACHE must be https://. The cache's certificate is issued by the
## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only
## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten
## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN —
## never added to the system trust store.
## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and
## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL
## publishes no repomd signature (404), so its metadata relies on HTTPS end
## to end (client->cache->dl.fedoraproject.org/mirrors, all verified).
## * release-install downloads epel-release / remi-release (from the cache, or
## from the HTTPS origin without PKG_CACHE), checks the package name, imports
## the signing keys committed in keys/pkg-cache/ and requires
## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1.
## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail
## a build that gets old PHP (replay of old-but-signed metadata).
##
## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if
## the cache dies mid-build dnf fails over to the origin by itself.
## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the
## cache, and a build without PKG_CACHE uses the upstream sources.
## cache (with the CA), and a build without PKG_CACHE uses upstream sources.
set -euo pipefail
MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-urls|restore}"
STATE=/var/lib/pkg-cache-repos # pristine copies of every file we rewrite
if [ "$MODE" = restore ]; then
[ -d "$STATE" ] || exit 0
(cd "$STATE" && find . -type f) | while read -r rel; do
cat "$STATE/$rel" > "/${rel#./}"
done
rm -rf "$STATE"
echo "pkg-cache-repos: original repository files restored"
exit 0
fi
# Keep the first-seen (pre-rewrite) copy of a file; later calls in the same RUN
# never overwrite it.
save() {
[ -e "$STATE$1" ] && return 0
mkdir -p "$STATE$(dirname "$1")"
cp -p "$1" "$STATE$1"
}
MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|restore}"
shift || true
STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files
SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-*
C="${PKG_CACHE:-}"
C="${C%/}"
if [ "$MODE" = release-urls ]; then
die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; }
if [ -n "$C" ]; then
case "$C" in
https://*) ;;
*) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;;
esac
HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}"
[ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing"
fi
# Files that may hold repo configuration; restore re-checks all of them.
repo_files() {
find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true
}
start_session() {
[ -d "$STATE" ] && return 0
mkdir -p "$STATE/orig"
repo_files > "$STATE/manifest" # what existed before this RUN touched anything
: > "$STATE/created"
printf '%s\n' "$HOSTPORT" > "$STATE/host"
}
save() { # keep the first-seen (pre-rewrite) copy
[ -e "$STATE/orig$1" ] && return 0
mkdir -p "$STATE/orig$(dirname "$1")"
cp -p "$1" "$STATE/orig$1"
}
created() { printf '%s\n' "$1" >> "$STATE/created"; }
if [ "$MODE" = restore ]; then
[ -d "$STATE" ] || exit 0
host=$(cat "$STATE/host")
(cd "$STATE/orig" && find . -type f) | while read -r rel; do
cat "$STATE/orig/$rel" > "/${rel#./}"
done
while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created"
# A repo/apt file that appeared during this RUN and still names the cache
# (e.g. written by a package from the rewritten repos) is removed too.
repo_files | while read -r f; do
if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then
echo "pkg-cache-repos: removing new file that references the cache: $f"
rm -f -- "$f"
fi
done
left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true)
rm -rf "$STATE"
[ -z "$left" ] || die "cache address $host still present after restore in: $left"
echo "pkg-cache-repos: original repository files restored (no reference to $host left)"
exit 0
fi
if [ "$MODE" = release-install ]; then
[ $# -gt 0 ] || die "release-install needs: epel and/or remi"
tmp=$(mktemp -d)
curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120)
if [ -n "$C" ]; then
echo "$C/epel/epel-release-latest-10.noarch.rpm $C/remi/enterprise/remi-release-10.rpm"
else
echo "https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm https://rpms.remirepo.net/enterprise/remi-release-10.rpm"
start_session
cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"
curl_opts+=(--cacert "$STATE/ca-bundle.pem")
fi
files=()
for name in "$@"; do
case "$name" in
epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-10
path=/epel/epel-release-latest-10.noarch.rpm
origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm ;;
remi) pkg=remi-release key=RPM-GPG-KEY-remi2024
path=/remi/enterprise/remi-release-10.rpm
origin=https://rpms.remirepo.net/enterprise/remi-release-10.rpm ;;
*) die "unknown release package '$name'" ;;
esac
url="$origin"; [ -n "$C" ] && url="$C$path"
f="$tmp/$pkg.rpm"
curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url"
rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key"
got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true)
[ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg"
chk=$(rpm -K "$f" 2>&1 || true)
[[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk"
echo "pkg-cache-repos: $pkg verified ($chk) from $url"
files+=("$f")
done
dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}"
rm -rf "$tmp"
exit 0
fi
@@ -65,31 +142,34 @@ if [ -z "$C" ]; then
echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories"
exit 0
fi
case "$C" in
http://*|https://*) ;;
*) echo "FATAL(pkg-cache-repos): PKG_CACHE must be an http(s) URL, got '$C'" >&2; exit 1 ;;
esac
rewrite_rpm() {
local f="$1"
grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net)/' "$f" || return 0
save "$f"
# Per [section]: a (commented or live) baseurl on a known origin becomes
# "baseurl=<cache>/<prefix>/<path> <origin>/<path>", and that section's
# mirrorlist/metalink is commented out (dnf prefers them over baseurl).
# Sections without a known origin (vault debug/source, ...) are untouched.
awk -v C="$C" '
# "baseurl=<cache>/<prefix>/<path> <origin>/<path>"; that section's
# mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it
# gets sslcacert=<system bundle + pkg-cache CA>, gpgcheck=1 and, for repos
# that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched.
awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" '
function flush( i) {
for (i = 1; i <= n; i++) {
if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i]
if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue
print buf[i]
}
n = 0; hit = 0
if (hit) {
print "sslcacert=" BUNDLE
print "gpgcheck=1"
if (signed) print "repo_gpgcheck=1"
}
n = 0; hit = 0; signed = 0
}
function rw(line, re, prefix, origin, p) {
function rw(line, re, prefix, origin, sig, p) {
if (match(line, re)) {
p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p)
hit = 1
hit = 1; if (sig) signed = 1
return "baseurl=" C "/" prefix "/" p " " origin "/" p
}
return line
@@ -98,9 +178,9 @@ rewrite_rpm() {
{
l = $0
if (l ~ /^#? ?baseurl=/) {
l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux")
l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel")
l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net")
l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1)
l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0)
l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1)
}
buf[++n] = l
}
@@ -121,16 +201,23 @@ rewrite_apt() {
case "$MODE" in
rpm)
start_session
cat /etc/pki/tls/certs/ca-bundle.crt "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"
for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done
echo "pkg-cache-repos(rpm): repos now via $C:"
grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /'
;;
apt)
start_session
conf=/etc/apt/apt.conf.d/99pkg-cache
# Trust the pkg-cache CA for the cache host ONLY (apt per-host option).
printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf"
created "$conf"
for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
[ -f "$f" ] && rewrite_apt "$f"
done
echo "pkg-cache-repos(apt): sources now via $C:"
grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | grep -F "$C/" | sort -u | sed 's/^/ /'
;;
*) echo "FATAL(pkg-cache-repos): unknown mode '$MODE'" >&2; exit 1 ;;
*) die "unknown mode '$MODE'" ;;
esac